Read all objects
Overview
Sometimes, you may wish to list every object in a class, such as for manual backup when the backup
feature is not suitable, or to export vectors or specific object properties. You may also wish to then restore these objects as well, to a different Weaviate instance for example.
The best way to do this is with the after
operator, also called the cursor API.
The after
operator is based on the order of IDs. No other ordering of data, such as sorting or searching, is possible.
Retrieve and restore objects
For classes where multi-tenancy is enabled, you will also need to specify the tenant name when reading or creating objects. See Manage data: multi-tenancy operations for details on how.
List every object
To list every object, use the after
operator to loop through the data as shown in the example below.
The example connects to a "source" instance at https://some-endpoint.weaviate.network
. It also defines a function that will fetch a group of objects (and their title
property) in the WineReview
class from using the ID of the last object retrieved as the cursor.
To retrieve (and restore) vectors as well, include the vector
parameter as shown in the highlighted lines.
- Python (v4)
- Python (v3)
- JavaScript/TypeScript
- Java
- Go
collection = client.collections.get("YourName")
for item in collection.iterator():
print(item.properties)
import weaviate
source_client = weaviate.Client(
url="https://some-endpoint.weaviate.network", # Replace with your endpoint
auth_client_secret=weaviate.AuthApiKey(api_key="YOUR-WEAVIATE-API-KEY"), # If auth enabled. Replace w/ your Weaviate instance API key
)
batch_size = 20
class_name = "WineReview"
class_properties = ["title"]
cursor = None
def get_batch_with_cursor(client, class_name, class_properties, batch_size, cursor=None):
query = (
client.query.get(class_name, class_properties)
# Optionally retrieve the vector embedding by adding `vector` to the _additional fields
.with_additional(["id vector"])
.with_limit(batch_size)
)
if cursor is not None:
return query.with_after(cursor).do()
else:
return query.do()
import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client';
const sourceClient = weaviate.client({
scheme: 'https',
host: 'some-endpoint.weaviate.network', // Replace with your Weaviate URL
apiKey: new ApiKey('YOUR-WEAVIATE-API-KEY'), // If auth enabled. Replace w/ your Weaviate instance API key.
});
const batchSize = 20;
const className = 'WineReview';
const classProperties = ['title'];
async function getBatchWithCursor(
client: WeaviateClient,
className: string, classProperties: string[],
batchSize: number, cursor?: string
): Promise<{ data: any }> {
const query = client.graphql.get()
.withClassName(className)
// Optionally retrieve the vector embedding by adding `vector` to the _additional fields
.withFields(classProperties.join(' ') + ' _additional { id vector }')
.withLimit(batchSize);
if (cursor) {
return await query.withAfter(cursor).do();
} else {
return await query.do();
}
}
import io.weaviate.client.Config;
import io.weaviate.client.WeaviateAuthClient;
import io.weaviate.client.WeaviateClient;
String scheme = "https";
String host = "some-endpoint.weaviate.network"; // Replace with your Weaviate URL
String apiKey = "YOUR-WEAVIATE-API-KEY"; // If auth enabled. Replace w/ your Weaviate instance API key.
try {
WeaviateClient sourceClient = WeaviateAuthClient.apiKey(new Config(scheme, host), apiKey);
} catch (AuthException e) {
// handle error in case of authorization problems
throw new RuntimeException(e);
}
int batchSize = 20;
String className = "WineReview";
String[] classProperties = new String[]{"title"};
private Result<GraphQLResponse> getBatchWithCursor(WeaviateClient client,
String className, String[] properties, int batchSize, String cursor) {
Get query = client.graphQL().get()
.withClassName(className)
// Optionally retrieve the vector embedding by adding `vector` to the _additional fields
.withFields(Stream.concat(Arrays.stream(properties), Stream.of("_additional { id vector }"))
.map(prop -> Field.builder().name(prop).build())
.toArray(Field[]::new)
)
.withLimit(batchSize);
if (cursor != null) {
return query.withAfter(cursor).run();
}
return query.run();
}
private List<Map<String, Object>> getProperties(GraphQLResponse result, String className, String[] classProperties) {
Object get = ((Map<?, ?>) result.getData()).get("Get");
Object clazz = ((Map<?, ?>) get).get(className);
List<?> objects = (List<?>) clazz;
List<Map<String, Object>> res = new ArrayList<>();
for (Object obj : objects) {
Map<String, Object> objProps = new HashMap<>();
for (String prop: classProperties) {
Object propValue = ((Map<?, ?>) obj).get(prop);
objProps.put(prop, propValue);
}
Object additional = ((Map<?, ?>) obj).get("_additional");
Object id = ((Map<?, ?>) additional).get("id");
objProps.put("id", id);
Object vector = ((Map<?, ?>) additional).get("vector");
objProps.put("vector", vector);
res.add(objProps);
}
return res;
}
private int getObjectsCount(GraphQLResponse result, String className) {
Object get = ((Map<?, ?>) result.getData()).get("Get");
Object clazz = ((Map<?, ?>) get).get(className);
List<?> objects = (List<?>) clazz;
return objects.size();
}
"github.com/weaviate/weaviate-go-client/v4/weaviate"
"github.com/weaviate/weaviate-go-client/v4/weaviate/auth"
"github.com/weaviate/weaviate-go-client/v4/weaviate/graphql"
"github.com/weaviate/weaviate/entities/models"
sourceClient, err := weaviate.NewClient(weaviate.Config{
Scheme: "https",
Host: "some-endpoint.weaviate.network", // Replace with your Weaviate URL
AuthConfig: auth.ApiKey{
Value: "YOUR-WEAVIATE-API-KEY", // If auth enabled. Replace w/ your Weaviate instance API key.
},
})
if err != nil {
// handle error
panic(err)
}
batchSize := 20
className := "WineReview"
classProperties := []string{"title"}
getBatchWithCursor := func(client weaviate.Client,
className string, classProperties []string, batchSize int, cursor string) (*models.GraphQLResponse, error) {
fields := []graphql.Field{}
for _, prop := range classProperties {
fields = append(fields, graphql.Field{Name: prop})
}
fields = append(fields, graphql.Field{Name: "_additional { id vector }"})
get := client.GraphQL().Get().
WithClassName(className).
// Optionally retrieve the vector embedding by adding `vector` to the _additional fields
WithFields(fields...).
WithLimit(batchSize)
if cursor != "" {
return get.WithAfter(cursor).Do(context.Background())
}
return get.Do(context.Background())
}
Fetch the class definition
You can fetch the existing class definition like this:
- Python (v4)
- Python (v3)
- JavaScript/TypeScript
- Java
- Go
# The example is coming soon
"Work in progress 👷🏻"
class_schema = source_client.schema.get(class_name)
const classDef = await sourceClient.schema.classGetter().withClassName(className).do();
Result<WeaviateClass> classDef = sourceClient.schema().classGetter()
.withClassName(className)
.run();
classDef, err := sourceClient.Schema().ClassGetter().WithClassName(className).Do(ctx)
Restore to a target instance
And then restore to a target instance, by:
- Creating the same class in the target instance using the fetched class definition, and
- Then streaming the objects from the source instance to the target instance using batch imports.
- Python (v4)
- Python (v3)
- JavaScript/TypeScript
- Java
- Go
# The example is coming soon
"Work in progress 👷🏻"
target_client = weaviate.Client(
url="https://anon-endpoint.weaviate.network", # Replace with your endpoint
)
target_client.schema.create_class(class_schema)
with target_client.batch(
batch_size=50,
) as batch:
# Batch import all objects to the target instance
while True:
# From the SOURCE instance, get the next group of objects
results = get_batch_with_cursor(source_client, class_name, class_properties, batch_size, cursor)
# If empty, we're finished
if len(results["data"]["Get"][class_name]) == 0:
break
# Otherwise, add the objects to the batch to be added to the target instance
objects_list = results["data"]["Get"][class_name]
aggregate_count += len(objects_list)
for retrieved_object in objects_list:
new_object = dict()
for prop in class_properties:
new_object[prop] = retrieved_object[prop]
batch.add_data_object(
new_object,
class_name=class_name,
# Can update the vector if it was included in _additional above
vector=retrieved_object['_additional']['vector']
)
# Update the cursor to the id of the last retrieved object
cursor = results["data"]["Get"][class_name][-1]["_additional"]["id"]
const targetClient = weaviate.client({
scheme: 'https',
host: 'anon-endpoint.weaviate.network', // Replace with your endpoint
});
await targetClient.schema.classCreator().withClass(classDef).do();
let targetBatcher = targetClient.batch.objectsBatcher();
let results, cursor;
// Batch import all objects to the target instance
while (true) {
// From the SOURCE instance, get the next group of objects
results = await getBatchWithCursor(sourceClient, className, classProperties, batchSize, cursor);
// If empty, we're finished
if (results.data.Get[className].length === 0)
break;
// Otherwise, add the objects to the batch to be added to the target instance
for (const retrievedObject of results.data.Get[className]) {
const newObject = {};
for (const prop of classProperties)
newObject[prop] = retrievedObject[prop];
targetBatcher = targetBatcher.withObject({
class: className,
properties: newObject,
// Can update the vector if it was included in _additional above
vector: retrievedObject['_additional']['vector'],
});
// When the batch counter reaches batchSize, push the objects to Weaviate
if (++aggregateCount % batchSize === 0) {
console.log(`Imported ${aggregateCount} objects...`);
// Flush the batch queue and restart it
const response = await targetBatcher.do();
targetBatcher = targetClient.batch.objectsBatcher();
// Handle errors
for (const r of response)
if (r.result.errors)
throw r.result.errors;
}
}
// Update the cursor to the id of the last retrieved object
cursor = results.data.Get[className].at(-1)['_additional']['id'];
}
// Flush any remaining objects
if (targetBatcher.payload().objects.length > 0)
await targetBatcher.do();
WeaviateClient targetClient = new WeaviateClient(
new Config("https", "some-endpoint.weaviate.network") // Replace with your Weaviate URL
);
ObjectsBatcher targetBatcher = targetClient.batch().objectsBatcher();
Result<GraphQLResponse> results = null;
String cursor = null;
// Batch import all objects to the target instance
while (true) {
// From the SOURCE instance, get the next group of objects
results = getBatchWithCursor(sourceClient, className, classProperties, batchSize, cursor);
GraphQLResponse result = results.getResult();
if (!results.hasErrors() && result != null) {
// If empty, we're finished
if (getObjectsCount(result, className) == 0) {
break;
}
// Otherwise, add the objects to the batch to be added to the target instance
List<Map<String, Object>> objectProperties = getProperties(result, className, classProperties);
for (Map<String, Object> objProp : objectProperties) {
WeaviateObject.WeaviateObjectBuilder objBuilder = WeaviateObject.builder();
String id = (String) objProp.get("id");
objBuilder.id(id);
// Can update the vector if it was included in _additional above
objBuilder.vector((Float[]) objProp.get("vector"));
Map<String, Object> properties = objProp.entrySet().stream()
.filter(p -> !(p.getKey().equals("id") || p.getKey().equals("vector")))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
objBuilder.properties(properties);
targetBatcher.withObject(objBuilder.build());
cursor = id;
}
Result<ObjectGetResponse[]> runBatchInsert = targetBatcher.run();
if (runBatchInsert.hasErrors()) {
// Handle errors
throw new RuntimeException(runBatchInsert.getError().toString());
}
System.out.printf("Imported %s objects...%n", runBatchInsert.getResult().length);
targetBatcher = targetClient.batch().objectsBatcher();
}
}
targetClient, err := weaviate.NewClient(weaviate.Config{
Scheme: "https",
Host: "some-endpoint.weaviate.network", // Replace with your Weaviate URL
})
if err != nil {
// handle error
panic(err)
}
targetClient.Schema().ClassCreator().WithClass(classDef).Do(ctx)
targetBatcher := targetClient.Batch().ObjectsBatcher()
cursor := ""
getObjectProperties := func(results *models.GraphQLResponse,
className string) ([]map[string]interface{}, error) {
if len(results.Errors) > 0 {
// handle errors
errorMessages := make([]string, len(results.Errors))
for i, gqlErr := range results.Errors {
errorMessages[i] = fmt.Sprintf("path: %v message: %v", gqlErr.Path, gqlErr.Message)
}
return nil, fmt.Errorf("error: %v", strings.Join(errorMessages, ", "))
}
get := results.Data["Get"].(map[string]interface{})
objects := get[className].([]interface{})
res := make([]map[string]interface{}, len(objects))
for i, obj := range objects {
res[i] = obj.(map[string]interface{})
}
return res, nil
}
mustGetVector := func(v interface{}) models.C11yVector {
vec, ok := v.([]interface{})
if ok {
vector := make(models.C11yVector, len(vec))
for i := range vec {
vector[i] = vec[i].(float32)
}
return vector
}
panic("argument is not an array")
}
// Batch import all objects to the target instance
for {
// From the SOURCE instance, get the next group of objects
results, err := getBatchWithCursor(*sourceClient, className, classProperties, batchSize, cursor)
if err != nil {
// handle errors
panic(err)
}
objectProperties, err := getObjectProperties(results, className)
if err != nil {
// handle errors
panic(err)
}
// If empty, we're finished
if len(objectProperties) == 0 {
break
}
// Otherwise, add the objects to the batch to be added to the target instance
for _, objProp := range objectProperties {
_additional := objProp["_additional"]
properties := map[string]interface{}{}
for name, value := range objProp {
if name != "_additional" {
properties[name] = value
}
}
targetBatcher.WithObjects(&models.Object{
Class: className,
Properties: properties,
// Can update the vector if it was included in _additional above
Vector: mustGetVector(_additional.(map[string]interface{})["vector"]),
})
}
resp, err := targetBatcher.Do(ctx)
if err != nil {
// handle errors
panic(err)
}
fmt.Printf("Imported %v objects...", len(resp))
}
Putting it together
Putting the pieces together, the example code retrieves the WineReview
class definition and objects from https://some-endpoint.weaviate.network
and uses them to populate the database at https://anon-endpoint.weaviate.network
:
- Python (v4)
- Python (v3)
- JavaScript/TypeScript
- Java
- Go
# The example is coming soon
"Work in progress 👷🏻"
# Retrieve data
import weaviate
source_client = weaviate.Client(
url="https://some-endpoint.weaviate.network", # Replace with your endpoint
auth_client_secret=weaviate.AuthApiKey(api_key="YOUR-WEAVIATE-API-KEY"), # If auth enabled. Replace w/ your Weaviate instance API key
)
batch_size = 20
class_name = "WineReview"
class_properties = ["title"]
cursor = None
def get_batch_with_cursor(client, class_name, class_properties, batch_size, cursor=None):
query = (
client.query.get(class_name, class_properties)
# Optionally retrieve the vector embedding by adding `vector` to the _additional fields
.with_additional(["id vector"])
.with_limit(batch_size)
)
if cursor is not None:
return query.with_after(cursor).do()
else:
return query.do()
# Use this function to retrieve data
# START FetchClassDefinition
class_schema = source_client.schema.get(class_name)
# END FetchClassDefinition
# Restore to a new (target) instance
target_client = weaviate.Client(
url="https://anon-endpoint.weaviate.network", # Replace with your endpoint
)
target_client.schema.create_class(class_schema)
with target_client.batch(
batch_size=50,
) as batch:
# Batch import all objects to the target instance
while True:
# From the SOURCE instance, get the next group of objects
results = get_batch_with_cursor(source_client, class_name, class_properties, batch_size, cursor)
# If empty, we're finished
if len(results["data"]["Get"][class_name]) == 0:
break
# Otherwise, add the objects to the batch to be added to the target instance
objects_list = results["data"]["Get"][class_name]
aggregate_count += len(objects_list)
for retrieved_object in objects_list:
new_object = dict()
for prop in class_properties:
new_object[prop] = retrieved_object[prop]
batch.add_data_object(
new_object,
class_name=class_name,
# Can update the vector if it was included in _additional above
vector=retrieved_object['_additional']['vector']
)
# Update the cursor to the id of the last retrieved object
cursor = results["data"]["Get"][class_name][-1]["_additional"]["id"]
import weaviate, { WeaviateClient, ApiKey } from 'weaviate-ts-client';
const sourceClient = weaviate.client({
scheme: 'https',
host: 'some-endpoint.weaviate.network', // Replace with your Weaviate URL
apiKey: new ApiKey('YOUR-WEAVIATE-API-KEY'), // If auth enabled. Replace w/ your Weaviate instance API key.
});
const batchSize = 20;
const className = 'WineReview';
const classProperties = ['title'];
async function getBatchWithCursor(
client: WeaviateClient,
className: string, classProperties: string[],
batchSize: number, cursor?: string
): Promise<{ data: any }> {
const query = client.graphql.get()
.withClassName(className)
// Optionally retrieve the vector embedding by adding `vector` to the _additional fields
.withFields(classProperties.join(' ') + ' _additional { id vector }')
.withLimit(batchSize);
if (cursor) {
return await query.withAfter(cursor).do();
} else {
return await query.do();
}
}
// Use this function to retrieve data
// START FetchClassDefinition
const classDef = await sourceClient.schema.classGetter().withClassName(className).do();
// END FetchClassDefinition
// Restore to a new (target) instance
const targetClient = weaviate.client({
scheme: 'https',
host: 'anon-endpoint.weaviate.network', // Replace with your endpoint
});
await targetClient.schema.classCreator().withClass(classDef).do();
let targetBatcher = targetClient.batch.objectsBatcher();
let results, cursor;
// Batch import all objects to the target instance
while (true) {
// From the SOURCE instance, get the next group of objects
results = await getBatchWithCursor(sourceClient, className, classProperties, batchSize, cursor);
// If empty, we're finished
if (results.data.Get[className].length === 0)
break;
// Otherwise, add the objects to the batch to be added to the target instance
for (const retrievedObject of results.data.Get[className]) {
const newObject = {};
for (const prop of classProperties)
newObject[prop] = retrievedObject[prop];
targetBatcher = targetBatcher.withObject({
class: className,
properties: newObject,
// Can update the vector if it was included in _additional above
vector: retrievedObject['_additional']['vector'],
});
// When the batch counter reaches batchSize, push the objects to Weaviate
if (++aggregateCount % batchSize === 0) {
console.log(`Imported ${aggregateCount} objects...`);
// Flush the batch queue and restart it
const response = await targetBatcher.do();
targetBatcher = targetClient.batch.objectsBatcher();
// Handle errors
for (const r of response)
if (r.result.errors)
throw r.result.errors;
}
}
// Update the cursor to the id of the last retrieved object
cursor = results.data.Get[className].at(-1)['_additional']['id'];
}
// Flush any remaining objects
if (targetBatcher.payload().objects.length > 0)
await targetBatcher.do();
import io.weaviate.client.Config;
import io.weaviate.client.WeaviateAuthClient;
import io.weaviate.client.WeaviateClient;
String scheme = "https";
String host = "some-endpoint.weaviate.network"; // Replace with your Weaviate URL
String apiKey = "YOUR-WEAVIATE-API-KEY"; // If auth enabled. Replace w/ your Weaviate instance API key.
try {
WeaviateClient sourceClient = WeaviateAuthClient.apiKey(new Config(scheme, host), apiKey);
} catch (AuthException e) {
// handle error in case of authorization problems
throw new RuntimeException(e);
}
int batchSize = 20;
String className = "WineReview";
String[] classProperties = new String[]{"title"};
private Result<GraphQLResponse> getBatchWithCursor(WeaviateClient client,
String className, String[] properties, int batchSize, String cursor) {
Get query = client.graphQL().get()
.withClassName(className)
// Optionally retrieve the vector embedding by adding `vector` to the _additional fields
.withFields(Stream.concat(Arrays.stream(properties), Stream.of("_additional { id vector }"))
.map(prop -> Field.builder().name(prop).build())
.toArray(Field[]::new)
)
.withLimit(batchSize);
if (cursor != null) {
return query.withAfter(cursor).run();
}
return query.run();
}
private List<Map<String, Object>> getProperties(GraphQLResponse result, String className, String[] classProperties) {
Object get = ((Map<?, ?>) result.getData()).get("Get");
Object clazz = ((Map<?, ?>) get).get(className);
List<?> objects = (List<?>) clazz;
List<Map<String, Object>> res = new ArrayList<>();
for (Object obj : objects) {
Map<String, Object> objProps = new HashMap<>();
for (String prop: classProperties) {
Object propValue = ((Map<?, ?>) obj).get(prop);
objProps.put(prop, propValue);
}
Object additional = ((Map<?, ?>) obj).get("_additional");
Object id = ((Map<?, ?>) additional).get("id");
objProps.put("id", id);
Object vector = ((Map<?, ?>) additional).get("vector");
objProps.put("vector", vector);
res.add(objProps);
}
return res;
}
private int getObjectsCount(GraphQLResponse result, String className) {
Object get = ((Map<?, ?>) result.getData()).get("Get");
Object clazz = ((Map<?, ?>) get).get(className);
List<?> objects = (List<?>) clazz;
return objects.size();
}
ObjectsBatcher targetBatcher = targetClient.batch().objectsBatcher();
Result<GraphQLResponse> results = null;
String cursor = null;
// Batch import all objects to the target instance
while (true) {
// From the SOURCE instance, get the next group of objects
results = getBatchWithCursor(sourceClient, className, classProperties, batchSize, cursor);
GraphQLResponse result = results.getResult();
if (!results.hasErrors() && result != null) {
// If empty, we're finished
if (getObjectsCount(result, className) == 0) {
break;
}
// Otherwise, add the objects to the batch to be added to the target instance
List<Map<String, Object>> objectProperties = getProperties(result, className, classProperties);
for (Map<String, Object> objProp : objectProperties) {
WeaviateObject.WeaviateObjectBuilder objBuilder = WeaviateObject.builder();
String id = (String) objProp.get("id");
objBuilder.id(id);
// Can update the vector if it was included in _additional above
objBuilder.vector((Float[]) objProp.get("vector"));
Map<String, Object> properties = objProp.entrySet().stream()
.filter(p -> !(p.getKey().equals("id") || p.getKey().equals("vector")))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
objBuilder.properties(properties);
targetBatcher.withObject(objBuilder.build());
cursor = id;
}
Result<ObjectGetResponse[]> runBatchInsert = targetBatcher.run();
if (runBatchInsert.hasErrors()) {
// Handle errors
throw new RuntimeException(runBatchInsert.getError().toString());
}
System.out.printf("Imported %s objects...%n", runBatchInsert.getResult().length);
targetBatcher = targetClient.batch().objectsBatcher();
}
}
"github.com/weaviate/weaviate-go-client/v4/weaviate"
"github.com/weaviate/weaviate-go-client/v4/weaviate/auth"
"github.com/weaviate/weaviate-go-client/v4/weaviate/graphql"
"github.com/weaviate/weaviate/entities/models"
sourceClient, err := weaviate.NewClient(weaviate.Config{
Scheme: "https",
Host: "some-endpoint.weaviate.network", // Replace with your Weaviate URL
AuthConfig: auth.ApiKey{
Value: "YOUR-WEAVIATE-API-KEY", // If auth enabled. Replace w/ your Weaviate instance API key.
},
})
if err != nil {
// handle error
panic(err)
}
batchSize := 20
className := "WineReview"
classProperties := []string{"title"}
getBatchWithCursor := func(client weaviate.Client,
className string, classProperties []string, batchSize int, cursor string) (*models.GraphQLResponse, error) {
fields := []graphql.Field{}
for _, prop := range classProperties {
fields = append(fields, graphql.Field{Name: prop})
}
fields = append(fields, graphql.Field{Name: "_additional { id vector }"})
get := client.GraphQL().Get().
WithClassName(className).
// Optionally retrieve the vector embedding by adding `vector` to the _additional fields
WithFields(fields...).
WithLimit(batchSize)
if cursor != "" {
return get.WithAfter(cursor).Do(context.Background())
}
return get.Do(context.Background())
}
// Use this function to retrieve data
// START FetchClassDefinition
classDef, err := sourceClient.Schema().ClassGetter().WithClassName(className).Do(ctx)
// END FetchClassDefinition
require.NoError(t, err)
// Restore to a new (target) instance
targetClient, err := weaviate.NewClient(weaviate.Config{
Scheme: "https",
Host: "some-endpoint.weaviate.network", // Replace with your Weaviate URL
})
if err != nil {
// handle error
panic(err)
}
targetClient.Schema().ClassCreator().WithClass(classDef).Do(ctx)
targetBatcher := targetClient.Batch().ObjectsBatcher()
cursor := ""
getObjectProperties := func(results *models.GraphQLResponse,
className string) ([]map[string]interface{}, error) {
if len(results.Errors) > 0 {
// handle errors
errorMessages := make([]string, len(results.Errors))
for i, gqlErr := range results.Errors {
errorMessages[i] = fmt.Sprintf("path: %v message: %v", gqlErr.Path, gqlErr.Message)
}
return nil, fmt.Errorf("error: %v", strings.Join(errorMessages, ", "))
}
get := results.Data["Get"].(map[string]interface{})
objects := get[className].([]interface{})
res := make([]map[string]interface{}, len(objects))
for i, obj := range objects {
res[i] = obj.(map[string]interface{})
}
return res, nil
}
mustGetVector := func(v interface{}) models.C11yVector {
vec, ok := v.([]interface{})
if ok {
vector := make(models.C11yVector, len(vec))
for i := range vec {
vector[i] = vec[i].(float32)
}
return vector
}
panic("argument is not an array")
}
// Batch import all objects to the target instance
for {
// From the SOURCE instance, get the next group of objects
results, err := getBatchWithCursor(*sourceClient, className, classProperties, batchSize, cursor)
if err != nil {
// handle errors
panic(err)
}
objectProperties, err := getObjectProperties(results, className)
if err != nil {
// handle errors
panic(err)
}
// If empty, we're finished
if len(objectProperties) == 0 {
break
}
// Otherwise, add the objects to the batch to be added to the target instance
for _, objProp := range objectProperties {
_additional := objProp["_additional"]
properties := map[string]interface{}{}
for name, value := range objProp {
if name != "_additional" {
properties[name] = value
}
}
targetBatcher.WithObjects(&models.Object{
Class: className,
Properties: properties,
// Can update the vector if it was included in _additional above
Vector: mustGetVector(_additional.(map[string]interface{})["vector"]),
})
}
resp, err := targetBatcher.Do(ctx)
if err != nil {
// handle errors
panic(err)
}
fmt.Printf("Imported %v objects...", len(resp))
}