Understand the Limitation of the String Property Type of Google Cloud Datastore

Google Cloud Datastore supports a variety of data types for property values:

  • Integers
  • Floating-point numbers
  • Strings
  • Dates
  • Binary data

Strings are likely the most frequently used. When working with strings, the most important question to ask is how many bytes can be inserted into the property/field. According to the table listed in properties and value types:

Up to 1500 bytes if property is indexed, up to 1MB otherwise

Let’s give a try:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
// String Property Type
// ====================
//
// Investigate the string property type of Google Cloud Datastore.
//
// Dependencies tested:
//
// - `gcloud@v0.21.0`
// - `lorem-ipsum@1.0.3`
'use strict';
// Define dependencies.
var params = {
projectId: 'myproj',
keyFilename: '/path/to/key/file.json',
namespace: 'test'
};
var dataset = require('gcloud').datastore.dataset(params);
var kind = 'String';
var lorem = require('lorem-ipsum');
// Save an entity to the datastore.
function run(params) {
var size = params.size;
var index = params.index;
var str = lorem({ count: size }).substr(0, size);
var key = dataset.key(kind);
var data = [
{
name: 'title',
value: str,
excludeFromIndexes: !index
}
];
var entity = {
key: key,
data: data
};
dataset.save(entity, function (err) {
console.log(size, new Buffer(str).length, index,
err ? err.code + ' ' + err.message : '200 OK');
});
}
// Explanation of fields:
//
// - `size`: Total number of bytes to produce
// - `index`: Whether to index the string field
//
// In-line comment indicates the expected result.
[
{ size: 1500, index: true }, // OK
{ size: 1501, index: false }, // OK
{ size: 1501, index: true }, // Error
{ size: 1024 * 1024 - 92, index: false }, // OK
{ size: 1024 * 1024 - 91, index: false }, // Error
].forEach(run);

Don’t ask me where 91 or 92 is coming from. Apparently, this is somewhere closer to 1 MB, but not exactly. The result of the test script: