Configure PQ index compression

Overview

Product quantization (PQ) is a form of data compression that reduces the memory footprint of a vector index. HNSW is an in-memory vector index, so enabling PQ for HNSW lets you work with larger datasets. For a discussion of how PQ saves memory, see this concepts section.

PQ makes tradeoffs between recall, performance, and memory usage. This means a PQ configuration that reduces memory may also reduce recall. There are similar trade-offs when you use HNSW without PQ. If you use PQ compression, you should also tune HNSW so that they compliment each other.

To configure HNSW, see Configuration: Indexes .

To learn how to configure PQ, follow the discussion on this page.

note

Before you enable PQ, be sure to provide a set of vectors to train the algorithm. For details, see Enable and train PQ

Prerequisites

This How-to page uses a dataset of 1000 Jeopardy questions. Download the data.

Python (v4)

import requests
import json

# Download the data
resp = requests.get(
    "https://raw.githubusercontent.com/weaviate-tutorials/intro-workshop/main/data/jeopardy_1k.json"
)

# Load the data so you can see what it is
data = json.loads(resp.text)

# Parse the JSON and preview it
print(type(data), len(data))
print(json.dumps(data[1], indent=2))

Python (v3)

import requests
import json

# Download the data
resp = requests.get(
    "https://raw.githubusercontent.com/weaviate-tutorials/intro-workshop/main/data/jeopardy_1k.json"
)

# Load the data so you can see what it is
data = json.loads(resp.text)

# Parse the JSON and preview it
print(type(data), len(data))
print(json.dumps(data[1], indent=2))

JavaScript/TypeScript

async function getJsonData() {
  const file = await fetch(
    'https://raw.githubusercontent.com/weaviate-tutorials/intro-workshop/main/data/jeopardy_1k.json'
  );
  return file.json();
}

await getJsonData();

Go

resp, err := http.Get(url)
if err != nil {
 log.Fatalf("get dataset: %v", err)
}

var data []struct {
  Question string `json:"Question"`
  Answer   string `json:"Answer"`
}
 
err = json.NewDecoder(resp.Body).Decode(&data)
if err != nil {
 log.Fatalf("read dataset: %v", err)
}

log.Printf("data: %+v", data)

Java

public static class Jeopardy {
  @SerializedName("Air Date")
  public Date airDate;
  @SerializedName("Round")
  public String round;
  @SerializedName("Value")
  public Integer value;
  @SerializedName("Category")
  public String category;
  @SerializedName("Question")
  public String question;
  @SerializedName("Answer")
  public String answer;
}

URL url = new URL("https://raw.githubusercontent.com/weaviate-tutorials/intro-workshop/main/data/jeopardy_1k.json");
InputStreamReader reader = new InputStreamReader(url.openStream());

Type listType = new TypeToken<List<Jeopardy>>(){}.getType();
List<Jeopardy> jeopardyList = new Gson().fromJson(reader, listType);

Enable PQ compression

To enable PQ compression, complete the following steps.

1. Connect to a Weaviate instance
2. Configure an initial schema without PQ
3. Load some training data
4. Enable and train PQ
5. Load the rest of your data

The next few sections work through these steps.

Step 1. Connect to a Weaviate instance

Use one of the Weaviate client libraries to connect to your instance.

After you install the client, connect to your instance.

Python (v4)

import weaviate, os, json
import weaviate.classes as wvc

client = weaviate.connect_to_local(
    headers={
        "X-OpenAI-Api-Key": os.environ[
            "OPENAI_API_KEY"
        ]  # Replace with your OpenAI API key
    }
)

client.is_ready()

Python (v3)

import weaviate
import os

client = weaviate.Client(
    url="http://localhost:8080/",  # Replace with your endpoint
    additional_headers={
        "X-OpenAI-Api-Key": os.getenv(
            "OPENAI_API_KEY"
        )  # Replace with your OpenAI API key
    },
)

print(client.is_ready())

JavaScript/TypeScript

import weaviate, { WeaviateClient, ObjectsBatcher } from 'weaviate-ts-client';
import fetch from 'node-fetch';

const client: WeaviateClient = weaviate.client({
  scheme: 'http',
  host: 'localhost:8080',
  headers: { 'X-OpenAI-Api-Key': process.env.OPENAI_APIKEY },  // Replace with your inference API key
});

Go

client, err := weaviate.NewClient(weaviate.Config{
  Scheme: "http",
  Host:   "localhost:8080",
  Headers: map[string]string{
   	"X-OpenAI-Api-Key": os.Getenv("OPENAI_APIKEY"),
  },
})

if err != nil {
   log.Fatalf("init client: %v", err)
}

Java

Config config = new Config("http", "localhost:8080", new HashMap<String, String>() {{
  put("X-Openai-Api-Key", "OPENAI_API_KEY"); // Replace with your OpenAI API key
}});

WeaviateClient client = new WeaviateClient(config);

Weaviate returns True if the connection is successful.

Step 2. Configure an initial schema without PQ

Every collection in your Weaviate instance is defined by a schema. This example defines a collection called Questions. Weaviate uses this schema during your initial data load. After the initial data set is loaded, you modify this schema to enable PQ.

Python (v4)

client.collections.create(
    name="Question",
    description="A Jeopardy! question",
    vectorizer_config=wvc.Configure.Vectorizer.text2vec_openai(),
    generative_config=wvc.Configure.Generative.openai(),
    properties=[
        wvc.Property(name="title", data_type=wvc.DataType.TEXT),
    ],
)

Python (v3)

class_definition = {
    "class": "Question",
    "vectorizer": "text2vec-openai",
    "properties": [
        {"name": "question", "dataType": ["text"]},
        {"name": "answer", "dataType": ["text"]},
        {"name": "round", "dataType": ["text"]},
    ],
}

client.schema.create_class(class_definition)

JavaScript/TypeScript

async function addSchema() {
  const classObj = {
    class: 'Question',
    vectorizer: 'text2vec-openai',
    properties: [
      { name: 'question', dataType: ['text'] },
      { name: 'answer', dataType: ['text'] },
    ],
  };
  const res = await client.schema.classCreator().withClass(classObj).do();
  console.log(res);
}

await addSchema();

Go

class := &models.Class{
  Class:      "Question",
  Vectorizer: "text2vec-openai",
  Properties: []*models.Property{
  		{Name: "Question", DataType: []string{"text"}},
  		{Name: "Answer", DataType: []string{"text"}},
  },
}

err = client.Schema().ClassCreator().
 WithClass(class).Do(context.Background())

if err != nil {
   log.Fatalf("create class: %v", err)
}

Java

List<Property> properties = Arrays.asList(
  Property.builder()
    .name("airDate")
    .dataType(Arrays.asList(DataType.DATE))
    .build(),
  Property.builder()
    .name("round")
    .dataType(Arrays.asList(DataType.TEXT))
    .build(),
  Property.builder()
    .name("value")
    .dataType(Arrays.asList(DataType.INT))
    .build(),
  Property.builder()
    .name("category")
    .dataType(Arrays.asList(DataType.TEXT))
    .build(),
  Property.builder()
    .name("question")
    .dataType(Arrays.asList(DataType.TEXT))
    .build(),
  Property.builder()
    .name("answer")
    .dataType(Arrays.asList(DataType.TEXT))
    .build()
);

WeaviateClass jeopardyClass = WeaviateClass.builder()
  .className("Question")
  .description("A Jeopardy! question")
  .properties(properties)
  .vectorizer("text2vec-openai")
  .build();

Result<Boolean> createResult = client.schema().classCreator()
  .withClass(jeopardyClass)
  .run();

Step 3. Load some training data

This example uses a relatively small data set to demonstrate loading data.

If you are starting with a new Weaviate instance, you should load between 10,000 and 100,000 objects from your data set. If you have multiple shards, you need to load between 10,000 and 100,000 objects on each shard.

You can use any of the objects in your data set. If possible, chose the objects at random so that they are independent and identically distributed.

By default Weaviate uses the first 100,000 objects in your database for the training step. If you have more than 100,000 objects Weaviate ignores the excess objects during the training period. However, the excess objects still take up memory. If you have a large dataset, consider training PQ on an initial set of 10,000 to 100,000 objects first and then uploading the rest of your data after PQ is enabled.

If you already have data in your Weaviate instance, you can move ahead to the next step.

Python (v4)

def parse_data():
    object_list = []
    for obj in data:
        object_list.append(
            {
                "question": obj["Question"],
                "answer": obj["Answer"],
                "round": obj["Round"],
            }
        )

    return object_list

jeopardy = client.collections.get("Question")
jeopardy.data.insert_many(parse_data())

# Check upload
response = jeopardy.aggregate.over_all(total_count=True)

# Should equal the number of objects uploaded
print(response.total_count)

Python (v3)

with client.batch as batch:
    for o in data:
        obj_body = {
            "question": o["Question"],
            "answer": o["Answer"],
            "round": o["Round"],
        }

        batch.add_data_object(data_object=obj_body, class_name="Question")

JavaScript/TypeScript

async function importQuestions() {
  // Get the questions directly from the URL
  const data = await getJsonData();

  // Prepare a batcher
  let batcher: ObjectsBatcher = client.batch.objectsBatcher();
  let counter = 0;
  const batchSize = 100;

  for (const question of data) {
    // Construct an object with a class and properties 'answer' and 'question'
    const obj = {
      class: 'Question',
      properties: {
        answer: question.Answer,
        question: question.Question,
      },
    };

    // add the object to the batch queue
    batcher = batcher.withObject(obj);

    // When the batch counter reaches batchSize, push the objects to Weaviate
    if (counter++ == batchSize) {
      // flush the batch queue
      const res = await batcher.do();
      // console.log(res);  // To view the response

      // restart the batch queue
      counter = 0;
      batcher = client.batch.objectsBatcher();
    }
  }

  // Flush the remaining objects
  const res = await batcher.do();
  // console.log(res);  // To view the response
}

await importQuestions();

Go

batcher := client.Batch().ObjectsBatcher()
for _, d := range data {
  batcher.WithObjects(&models.Object{
  		Class: "Question",
  		Properties: map[string]interface{}{
  				"question": d.Question,
  				"answer":   d.Answer,
  		},
  })
}

batch, err := batcher.Do(context.Background())
if err != nil {
   log.Fatalf("batcher: %v", err)
}

for i, b := range batch {
  if b.Result.Errors != nil {
  		for _, err := range b.Result.Errors.Error {
  			 log.Print(err.Message)
  	 	}
  		log.Fatalf("object@idx %d had batch errors above", i)
  }
}

Java

ObjectsBatcher batcher = client.batch().objectsAutoBatcher();

jeopardyList.forEach(jeopardy -> {
  WeaviateObject object = WeaviateObject.builder()
    .className("Question")
    .properties(new HashMap<String, Object>() {{
      put("airDate", DateFormatUtils.format(jeopardy.airDate, "yyyy-MM-dd'T'HH:mm:ssZZZZZ"));
      put("round", jeopardy.round);
      put("value", jeopardy.value);
      put("category", jeopardy.category);
      put("question", jeopardy.question);
      put("answer", jeopardy.answer);
    }})
    .build();

  batcher.withObject(object);
});

batcher.flush();
batcher.close();

Step 4. Enable and train PQ

You can enable PQ compression by changing the relevant configuration at the collection (i.e. class) level.

PQ relies on a codebook to compress the original vectors. The codebook defines "centroids" that are used to calculate the compressed vector. Weaviate’s PQ implementation uses existing data to train the codebook. You must have some vectors loaded before you enable PQ so Weaviate can use them to define the centroids. You should have 10,000 to 100,000 vectors loaded before you enable PQ.

After you update the schema, Weaviate trains PQ on the first 100,000 objects in your database. To use a different value, set a new trainingLimit. If you increase trainingLimit, the training period will take longer. You could also have memory problems if you set a high trainingLimit.

To change the compression rate, specify the number of segments. The number of vector dimensions must be evenly divisible by the number of segments. Fewer segments means smaller quantized vectors.

For additional configuration options, see the parameter table.

To enable PQ, update your schema as shown below.

Python (v4)

import weaviate.classes as wvc

jeopardy = client.collections.get("Question")
jeopardy.config.update(
    vector_index_config=wvc.Reconfigure.vector_index(
        pq_enabled=True, pq_segments=96, pq_training_limit=100000
    )
)

Python (v3)

client.schema.update_config(
    "Question",
    {
        "vectorIndexConfig": {
            "pq": {
                "enabled": True,  # Enable PQ
                "trainingLimit": 100000,
                "segments": 96
            }
        }
    },
)

JavaScript/TypeScript

// Note: This is carried out using the REST endpoint directly,
// as the JS/TS client cannot currently update the schema.
async function updateSchema() {
  let url = 'http://localhost:8080/v1/schema/Question';
  let pqUpdateDefinition = {
    class: 'Question',
    vectorizer: 'text2vec-openai',
    properties: [
      { name: 'question', dataType: ['text'] },
      { name: 'answer', dataType: ['text'] },
    ],
    vectorIndexConfig: {
      pq: {
        enabled: true,
        trainingLimit: 100000,
        segments: 96,
      },
    },
  };

  let response = await fetch(url, {
    method: 'PUT',
    headers: {
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(pqUpdateDefinition),
  });
  console.log(await response.json());
}

await updateSchema();

Go

class, err = client.Schema().ClassGetter().
   WithClassName("Question").Do(context.Background())

if err != nil {
   log.Fatalf("get class for vec idx cfg update: %v", err)
}

cfg := class.VectorIndexConfig.(map[string]interface{})
cfg["pq"] = map[string]interface{}{
  "enabled":       true,
  "trainingLimit": 100_000,
  "segments":      96,
}
class.VectorIndexConfig = cfg

err = client.Schema().ClassUpdater().
 WithClass(class).Do(context.Background())

if err != nil {
   log.Fatalf("update class to use pq: %v", err)
}

Java

WeaviateClass updatedJeopardyClass = WeaviateClass.builder()
  .className("Question")
  .description("A Jeopardy! question")
  .properties(properties)
  .vectorizer("text2vec-openai")
  .vectorIndexConfig(VectorIndexConfig.builder()
    .pq(PQConfig.builder()
      .enabled(true)
      .trainingLimit(100_000)
      .segments(96)
      .build())
    .build())
  .build();

Result<Boolean> updateResult = client.schema().classUpdater()
  .withClass(updatedJeopardyClass)
  .run();

Step 5. Load the rest of your data

If you are starting with a new Weaviate instance, you can load the rest of your data now. Weaviate compresses the new data when it adds it to the database.

If you already have data in your Weaviate instance, Weaviate automatically compresses the remaining objects (the ones after the initial training set).

PQ Parameters

You can configure PQ compression by setting the following parameters at the collection level.

Parameter	Type	Default	Details
enabled	boolean	false	Enable PQ. Weaviate use product quantization (PQ) compression when true.
trainingLimit	integer	100000	Object limit. The maximum number of objects, per shard, used to fit the centroids. Larger values increase the time it takes to fit the centroids. Larger values also require more memory.
segments	integer	--	The number of segments to use. By default segments is equal to the number of vector dimensions. Reducing the number of segments reduces the size of the quantized (PQ compressed) vectors. The number of vector dimensions must be evenly divisible by the number of segments.
centroids	integer	256	The number of centroids to use. Reducing the number of centroids reduces the size of the quantized (PQ compressed) vectors at the price of recall. If you use the kmeans encoder, centroids is set to 256 (one byte) by default.
encoder	string	kmeans	Encoder specification. There are two encoders. You can specify the type of encoder as either kmeans(default) or tile.
distribution	string	log-normal	Encoder distribution type. Only used with the tile encoder. If you use the tile encoder, you can specify the distribution as log-normal (default) or normal.

Additional tools and considerations

Check the system logs

When compression is enabled, Weaviate logs diagnostic messages like these.

pq-conf-demo-1  | {"action":"compress","level":"info","msg":"switching to compressed vectors","time":"2023-11-13T21:10:52Z"}

pq-conf-demo-1  | {"action":"compress","level":"info","msg":"vector compression complete","time":"2023-11-13T21:10:53Z"}

If you use docker-compose to run Weaviate, you can get the logs on the system console.

docker compose logs -f --tail 10 weaviate

You can also view the log file directly. Check docker to get the file location.

docker inspect --format='{{.LogPath}}' <your-weaviate-container-id>

Review the current pq configuration

To review the current pq configuration, you can retrieve it as shown below.

Python (v4)

jeopardy = client.collections.get("Question")
config = jeopardy.config.get()
pq_config = config.vector_index_config.pq

# print some of the config properties
print(f"Enabled: { pq_config.enabled }")
print(f"Training: { pq_config.training_limit }")
print(f"Segments: { pq_config.segments }")
print(f"Centroids: { pq_config.centroids }")

Python (v3)

schema = client.schema.get("Question")
print(json.dumps(schema["vectorIndexConfig"]["pq"], indent=2))

JavaScript/TypeScript

const schema = await client
  .schema
  .classGetter()
  .withClassName('Question')
  .do();

// Inspect the PQ configuration
console.log(JSON.stringify(schema.vectorIndexConfig.pq, null, 2));

Go

class, err = client.Schema().ClassGetter().
   WithClassName("Question").Do(context.Background())
if err != nil {
   log.Fatalf("get class to verify vec idx cfg changes: %v", err)
}

cfg = class.VectorIndexConfig.(map[string]interface{})
log.Printf("pq config: %v", cfg["pq"])

Java

Result<WeaviateClass> getResult = client.schema().classGetter()
  .withClassName("Question")
  .run();

PQConfig pqConfig = getResult.getResult().getVectorIndexConfig().getPq();
String json = new GsonBuilder().setPrettyPrinting().create().toJson(pqConfig);
System.out.println(json);

Related pages

• Configuration: Indexes
• Tutorial: Schema