Skip to content

Commit a5355e2

Browse files
tsmith023tsmith023
authored
Merge pull request #183 from weaviate/fix/merge-with-existing-in-collection-update
Fix/merge with existing in collection update
2 parents b2dc636 + a356e68 commit a5355e2

File tree

6 files changed

+78
-106
lines changed

6 files changed

+78
-106
lines changed

‎src/collections/config/classes.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ export class MergeWithExisting {
3838
if (update.vectorizers !== undefined) {
3939
if (Array.isArray(update.vectorizers)) {
4040
current.vectorConfig = MergeWithExisting.vectors(current.vectorConfig, update.vectorizers);
41-
} else if (supportsNamedVectors) {
41+
} else if (supportsNamedVectors && current.vectorConfig !== undefined) {
4242
const updateVectorizers = {
4343
...update.vectorizers,
4444
name: 'default',

‎src/collections/config/index.ts

+7-2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ import {
1616
CollectionConfig,
1717
CollectionConfigUpdate,
1818
PQConfig,
19+
QuantizerConfig,
20+
SQConfig,
1921
VectorIndexConfig,
2022
VectorIndexConfigDynamic,
2123
VectorIndexConfigFlat,
@@ -163,12 +165,15 @@ export class VectorIndex {
163165
}
164166

165167
export class Quantizer {
166-
static isPQ(config?: PQConfig | BQConfig): config is PQConfig {
168+
static isPQ(config?: QuantizerConfig): config is PQConfig {
167169
return config?.type === 'pq';
168170
}
169-
static isBQ(config?: PQConfig | BQConfig): config is BQConfig {
171+
static isBQ(config?: QuantizerConfig): config is BQConfig {
170172
return config?.type === 'bq';
171173
}
174+
static isSQ(config?: QuantizerConfig): config is SQConfig {
175+
return config?.type === 'sq';
176+
}
172177
}
173178

174179
export const configGuards = {

‎src/collections/config/integration.test.ts

+56-68
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/* eslint-disable @typescript-eslint/no-non-null-assertion */
22
import { WeaviateUnsupportedFeatureError } from '../../errors.js';
3-
import weaviate, { WeaviateClient } from '../../index.js';
3+
import weaviate, { WeaviateClient, weaviateV2 } from '../../index.js';
44
import { PropertyConfig, VectorIndexConfigDynamic, VectorIndexConfigHNSW } from './types/index.js';
55

66
const fail = (msg: string) => {
@@ -526,71 +526,59 @@ describe('Testing of the collection.config namespace', () => {
526526
expect(config.multiTenancy.enabled).toEqual(true);
527527
});
528528

529-
// it('should be able update the config of a collection with legacy vectors', async () => {
530-
// const collectionName = 'TestCollectionConfigUpdateLegacyVectors';
531-
// const collection = await client.collections.create({
532-
// name: collectionName,
533-
// properties: [
534-
// {
535-
// name: 'testProp',
536-
// dataType: 'text',
537-
// },
538-
// ],
539-
// vectorizer: {
540-
// name: 'none',
541-
// config: {},
542-
// },
543-
// });
544-
// const config = await collection.config
545-
// .update({
546-
// vectorizers: weaviate.reconfigure.vectorIndex.hnsw({
547-
// quantizer: weaviate.reconfigure.vectorIndex.quantizer.pq(),
548-
// ef: 4,
549-
// }),
550-
// })
551-
// .then(() => collection.config.get());
552-
553-
// expect(config.name).toEqual(collectionName);
554-
// expect(config.properties).toEqual<PropertyConfig[]>([
555-
// {
556-
// name: 'testProp',
557-
// dataType: 'text',
558-
// description: undefined,
559-
// indexSearchable: true,
560-
// indexFilterable: true,
561-
// indexInverted: false,
562-
// vectorizerConfig: undefined,
563-
// nestedProperties: undefined,
564-
// tokenization: 'word',
565-
// },
566-
// ]);
567-
// expect(config.generative).toBeUndefined();
568-
// expect(config.reranker).toBeUndefined();
569-
// expect(config.vectorizers.default.indexConfig).toEqual<VectorIndexConfigHNSW>({
570-
// skip: false,
571-
// cleanupIntervalSeconds: 300,
572-
// maxConnections: 64,
573-
// efConstruction: 128,
574-
// ef: 4,
575-
// dynamicEfMin: 100,
576-
// dynamicEfMax: 500,
577-
// dynamicEfFactor: 8,
578-
// vectorCacheMaxObjects: 1000000000000,
579-
// flatSearchCutoff: 40000,
580-
// distance: 'cosine',
581-
// quantizer: {
582-
// bitCompression: false,
583-
// segments: 0,
584-
// centroids: 256,
585-
// trainingLimit: 100000,
586-
// encoder: {
587-
// type: 'kmeans',
588-
// distribution: 'log-normal',
589-
// },
590-
// type: 'pq',
591-
// },
592-
// });
593-
// expect(config.vectorizers.default.indexType).toEqual('hnsw');
594-
// expect(config.vectorizers.default.vectorizer.name).toEqual('none');
595-
// });
529+
it('should be able update the config of a collection with legacy vectors', async () => {
530+
const clientV2 = weaviateV2.client({
531+
host: 'http://localhost:8080',
532+
});
533+
const collectionName = 'TestCollectionConfigUpdateLegacyVectors';
534+
await clientV2.schema
535+
.classCreator()
536+
.withClass({
537+
class: collectionName,
538+
vectorizer: 'none',
539+
})
540+
.do();
541+
const collection = client.collections.get(collectionName);
542+
const config = await collection.config
543+
.update({
544+
vectorizers: weaviate.reconfigure.vectorizer.update({
545+
vectorIndexConfig: weaviate.reconfigure.vectorIndex.hnsw({
546+
quantizer: weaviate.reconfigure.vectorIndex.quantizer.pq(),
547+
ef: 4,
548+
}),
549+
}),
550+
})
551+
.then(() => collection.config.get());
552+
553+
expect(config.name).toEqual(collectionName);
554+
expect(config.generative).toBeUndefined();
555+
expect(config.reranker).toBeUndefined();
556+
expect(config.vectorizers.default.indexConfig).toEqual<VectorIndexConfigHNSW>({
557+
skip: false,
558+
cleanupIntervalSeconds: 300,
559+
maxConnections: (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 26, 0))) ? 64 : 32,
560+
efConstruction: 128,
561+
ef: 4,
562+
dynamicEfMin: 100,
563+
dynamicEfMax: 500,
564+
dynamicEfFactor: 8,
565+
vectorCacheMaxObjects: 1000000000000,
566+
flatSearchCutoff: 40000,
567+
distance: 'cosine',
568+
type: 'hnsw',
569+
quantizer: {
570+
bitCompression: false,
571+
segments: 0,
572+
centroids: 256,
573+
trainingLimit: 100000,
574+
encoder: {
575+
type: 'kmeans',
576+
distribution: 'log-normal',
577+
},
578+
type: 'pq',
579+
},
580+
});
581+
expect(config.vectorizers.default.indexType).toEqual('hnsw');
582+
expect(config.vectorizers.default.vectorizer.name).toEqual('none');
583+
});
596584
});

‎src/collections/config/types/vectorIndex.ts

+2
Original file line numberDiff line numberDiff line change
@@ -73,3 +73,5 @@ export type PQEncoderDistribution = 'log-normal' | 'normal';
7373
export type VectorIndexType = 'hnsw' | 'flat' | 'dynamic' | string;
7474

7575
export type VectorIndexConfig = VectorIndexConfigHNSW | VectorIndexConfigFlat | VectorIndexConfigDynamic;
76+
77+
export type QuantizerConfig = PQConfig | BQConfig | SQConfig;

‎src/collections/configure/parsing.ts

-18
Original file line numberDiff line numberDiff line change
@@ -39,21 +39,3 @@ export class QuantizerGuards {
3939
export function parseWithDefault<D>(value: D | undefined, defaultValue: D): D {
4040
return value !== undefined ? value : defaultValue;
4141
}
42-
43-
export const parseQuantizer = <T extends QuantizerConfig>(config?: T): T | undefined => {
44-
if (config === undefined) {
45-
return undefined;
46-
}
47-
if (QuantizerGuards.isPQCreate(config)) {
48-
return {
49-
...config,
50-
type: 'pq',
51-
} as T;
52-
} else if (QuantizerGuards.isBQCreate(config)) {
53-
return {
54-
...config,
55-
type: 'bq',
56-
} as T;
57-
}
58-
return config;
59-
};

‎src/collections/configure/vectorIndex.ts

+12-17
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@ import {
1616
VectorIndexConfigHNSWUpdate,
1717
} from './types/index.js';
1818

19-
import { parseQuantizer } from './parsing.js';
20-
2119
const isModuleConfig = <N, C>(config: ModuleConfig<N, C> | C): config is ModuleConfig<N, C> => {
2220
return config && typeof config === 'object' && 'name' in config && 'config' in config;
2321
};
@@ -40,7 +38,7 @@ const configure = {
4038
config: {
4139
distance,
4240
vectorCacheMaxObjects,
43-
quantizer: parseQuantizer(quantizer),
41+
quantizer: quantizer,
4442
},
4543
};
4644
},
@@ -62,7 +60,7 @@ const configure = {
6260
? {
6361
...rest,
6462
distance: distanceMetric,
65-
quantizer: parseQuantizer(rest.quantizer),
63+
quantizer: rest.quantizer,
6664
}
6765
: undefined,
6866
};
@@ -177,10 +175,7 @@ const reconfigure = {
177175
}): ModuleConfig<'flat', VectorIndexConfigFlatUpdate> => {
178176
return {
179177
name: 'flat',
180-
config: {
181-
vectorCacheMaxObjects: options.vectorCacheMaxObjects,
182-
quantizer: parseQuantizer(options.quantizer),
183-
},
178+
config: options,
184179
};
185180
},
186181
/**
@@ -221,8 +216,8 @@ const reconfigure = {
221216
* NOTE: If the vector index already has a quantizer configured, you cannot change its quantizer type; only its values.
222217
* So if you want to change the quantizer type, you must recreate the collection.
223218
*
224-
* @param {boolean} [options.cache] Whether to cache the quantizer. Default is false.
225-
* @param {number} [options.rescoreLimit] The rescore limit. Default is 1000.
219+
* @param {boolean} [options.cache] Whether to cache the quantizer.
220+
* @param {number} [options.rescoreLimit] The new rescore limit.
226221
* @returns {BQConfigCreate} The configuration object.
227222
*/
228223
bq: (options?: { cache?: boolean; rescoreLimit?: number }): BQConfigUpdate => {
@@ -237,11 +232,11 @@ const reconfigure = {
237232
* NOTE: If the vector index already has a quantizer configured, you cannot change its quantizer type; only its values.
238233
* So if you want to change the quantizer type, you must recreate the collection.
239234
*
240-
* @param {number} [options.centroids] The number of centroids. Default is 256.
241-
* @param {PQEncoderDistribution} [options.pqEncoderDistribution] The encoder distribution. Default is 'log-normal'.
242-
* @param {PQEncoderType} [options.pqEncoderType] The encoder type. Default is 'kmeans'.
243-
* @param {number} [options.segments] The number of segments. Default is 0.
244-
* @param {number} [options.trainingLimit] The training limit. Default is 100000.
235+
* @param {number} [options.centroids] The new number of centroids.
236+
* @param {PQEncoderDistribution} [options.pqEncoderDistribution] The new encoder distribution.
237+
* @param {PQEncoderType} [options.pqEncoderType] The new encoder type.
238+
* @param {number} [options.segments] The new number of segments.
239+
* @param {number} [options.trainingLimit] The new training limit.
245240
* @returns {PQConfigUpdate} The configuration object.
246241
*/
247242
pq: (options?: {
@@ -270,8 +265,8 @@ const reconfigure = {
270265
* NOTE: If the vector index already has a quantizer configured, you cannot change its quantizer type; only its values.
271266
* So if you want to change the quantizer type, you must recreate the collection.
272267
*
273-
* @param {number} [options.rescoreLimit] The rescore limit. Default is 1000.
274-
* @param {number} [options.trainingLimit] The training limit. Default is 100000.
268+
* @param {number} [options.rescoreLimit] The rescore limit.
269+
* @param {number} [options.trainingLimit] The training limit.
275270
* @returns {SQConfigUpdate} The configuration object.
276271
*/
277272
sq: (options?: { rescoreLimit?: number; trainingLimit?: number }): SQConfigUpdate => {

0 commit comments

Comments
 (0)