Module org.elasticsearch.server

Package org.elasticsearch.index.codec.vectors.diskbbq

Class IVFVectorsWriter

java.lang.Object

org.apache.lucene.codecs.KnnVectorsWriter

org.elasticsearch.index.codec.vectors.diskbbq.IVFVectorsWriter

All Implemented Interfaces:

Closeable, AutoCloseable, org.apache.lucene.util.Accountable

Direct Known Subclasses:

ES920DiskBBQVectorsWriter, ES940DiskBBQVectorsWriter, ESNextDiskBBQVectorsWriter

public abstract class IVFVectorsWriter
extends org.apache.lucene.codecs.KnnVectorsWriter

Base class for IVF vectors writer.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static final record	IVFVectorsWriter.CentroidOffsetAndLength

Nested classes/interfaces inherited from class org.apache.lucene.codecs.KnnVectorsWriter

org.apache.lucene.codecs.KnnVectorsWriter.MergedVectorValues

Field Summary

Fields inherited from interface org.apache.lucene.util.Accountable

NULL_ACCOUNTABLE

Constructor Summary

Constructors

Modifier	Constructor	Description
protected	IVFVectorsWriter(org.apache.lucene.index.SegmentWriteState state, String rawVectorFormatName, Boolean useDirectIOReads, org.apache.lucene.codecs.hnsw.FlatVectorsWriter rawVectorDelegate, int writeVersion, String codecName, String metaExtension, String centroidExtension, String clusterExtension, boolean shouldWriteDirectIoReads, int flatVectorThreshold)

Method Summary

Modifier and Type	Method	Description
final org.apache.lucene.codecs.KnnFieldVectorsWriter<?>	addField(org.apache.lucene.index.FieldInfo fieldInfo)
abstract IVFVectorsWriter.CentroidOffsetAndLength	buildAndWritePostingsLists(org.apache.lucene.index.FieldInfo fieldInfo, CentroidSupplier centroidSupplier, org.apache.lucene.index.FloatVectorValues floatVectorValues, org.apache.lucene.store.IndexOutput postingsOutput, long fileOffset, int[] assignments, int[] overspillAssignments)
abstract IVFVectorsWriter.CentroidOffsetAndLength	buildAndWritePostingsLists(org.apache.lucene.index.FieldInfo fieldInfo, CentroidSupplier centroidSupplier, org.apache.lucene.index.FloatVectorValues floatVectorValues, org.apache.lucene.store.IndexOutput postingsOutput, long fileOffset, org.apache.lucene.index.MergeState mergeState, int[] assignments, int[] overspillAssignments)
abstract CentroidAssignments	calculateCentroids(org.apache.lucene.index.FieldInfo fieldInfo, KMeansFloatVectorValues floatVectorValues)
abstract CentroidAssignments	calculateCentroids(org.apache.lucene.index.FieldInfo fieldInfo, KMeansFloatVectorValues floatVectorValues, org.apache.lucene.index.MergeState mergeState)
final void	close()
abstract CentroidSupplier	createCentroidSupplier(org.apache.lucene.index.FieldInfo info, float[][] centroids, float[] globalCentroid)
abstract CentroidSupplier	createCentroidSupplier(org.apache.lucene.store.IndexInput centroidsInput, int numCentroids, org.apache.lucene.index.FieldInfo fieldInfo, float[] globalCentroid)
protected abstract Preconditioner	createPreconditioner(int dimension)
protected abstract void	doWriteMeta(org.apache.lucene.store.IndexOutput metaOutput, org.apache.lucene.index.FieldInfo field, int numCentroids, long preconditionerOffset, long preconditionerLength)
final void	finish()
final void	flush(int maxDoc, org.apache.lucene.index.Sorter.DocMap sortMap)
protected abstract Preconditioner	inheritPreconditioner(org.apache.lucene.index.FieldInfo fieldInfo, org.apache.lucene.index.MergeState mergeState)
final void	mergeOneField(org.apache.lucene.index.FieldInfo fieldInfo, org.apache.lucene.index.MergeState mergeState)
protected abstract Consumer<List<float[]>>	preconditionVectors(Preconditioner preconditioner)
protected abstract org.apache.lucene.index.FloatVectorValues	preconditionVectors(Preconditioner precondtioner, org.apache.lucene.index.FloatVectorValues vectors)
final long	ramBytesUsed()
abstract void	writeCentroids(org.apache.lucene.index.FieldInfo fieldInfo, CentroidSupplier centroidSupplier, int[] centroidAssignments, float[] globalCentroid, IVFVectorsWriter.CentroidOffsetAndLength centroidOffsetAndLength, org.apache.lucene.store.IndexOutput centroidOutput)
abstract void	writeCentroids(org.apache.lucene.index.FieldInfo fieldInfo, CentroidSupplier centroidSupplier, int[] centroidAssignments, float[] globalCentroid, IVFVectorsWriter.CentroidOffsetAndLength centroidOffsetAndLength, org.apache.lucene.store.IndexOutput centroidOutput, org.apache.lucene.index.MergeState mergeState)
protected abstract void	writePreconditioner(Preconditioner precondtioner, org.apache.lucene.store.IndexOutput out)

Methods inherited from class org.apache.lucene.codecs.KnnVectorsWriter

mapOldOrdToNewOrd, merge

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.lucene.util.Accountable

getChildResources

Constructor Details

IVFVectorsWriter

protected IVFVectorsWriter(org.apache.lucene.index.SegmentWriteState state,
 String rawVectorFormatName,
 Boolean useDirectIOReads,
 org.apache.lucene.codecs.hnsw.FlatVectorsWriter rawVectorDelegate,
 int writeVersion,
 String codecName,
 String metaExtension,
 String centroidExtension,
 String clusterExtension,
 boolean shouldWriteDirectIoReads,
 int flatVectorThreshold)
                    throws IOException

Throws:

IOException

Method Details

addField

public final org.apache.lucene.codecs.KnnFieldVectorsWriter<?> addField(org.apache.lucene.index.FieldInfo fieldInfo)
                                                                 throws IOException

Specified by:

addField in class org.apache.lucene.codecs.KnnVectorsWriter

Throws:

IOException

calculateCentroids

public abstract CentroidAssignments calculateCentroids(org.apache.lucene.index.FieldInfo fieldInfo,
 KMeansFloatVectorValues floatVectorValues)
                                                throws IOException

Throws:

IOException

calculateCentroids

public abstract CentroidAssignments calculateCentroids(org.apache.lucene.index.FieldInfo fieldInfo,
 KMeansFloatVectorValues floatVectorValues,
 org.apache.lucene.index.MergeState mergeState)
                                                throws IOException

Throws:

IOException

writeCentroids

public abstract void writeCentroids(org.apache.lucene.index.FieldInfo fieldInfo,
 CentroidSupplier centroidSupplier,
 int[] centroidAssignments,
 float[] globalCentroid,
 IVFVectorsWriter.CentroidOffsetAndLength centroidOffsetAndLength,
 org.apache.lucene.store.IndexOutput centroidOutput)
                             throws IOException

Throws:

IOException

writeCentroids

public abstract void writeCentroids(org.apache.lucene.index.FieldInfo fieldInfo,
 CentroidSupplier centroidSupplier,
 int[] centroidAssignments,
 float[] globalCentroid,
 IVFVectorsWriter.CentroidOffsetAndLength centroidOffsetAndLength,
 org.apache.lucene.store.IndexOutput centroidOutput,
 org.apache.lucene.index.MergeState mergeState)
                             throws IOException

Throws:

IOException

buildAndWritePostingsLists

public abstract IVFVectorsWriter.CentroidOffsetAndLength buildAndWritePostingsLists(org.apache.lucene.index.FieldInfo fieldInfo,
 CentroidSupplier centroidSupplier,
 org.apache.lucene.index.FloatVectorValues floatVectorValues,
 org.apache.lucene.store.IndexOutput postingsOutput,
 long fileOffset,
 int[] assignments,
 int[] overspillAssignments)
                                                                             throws IOException

Throws:

IOException

buildAndWritePostingsLists

public abstract IVFVectorsWriter.CentroidOffsetAndLength buildAndWritePostingsLists(org.apache.lucene.index.FieldInfo fieldInfo,
 CentroidSupplier centroidSupplier,
 org.apache.lucene.index.FloatVectorValues floatVectorValues,
 org.apache.lucene.store.IndexOutput postingsOutput,
 long fileOffset,
 org.apache.lucene.index.MergeState mergeState,
 int[] assignments,
 int[] overspillAssignments)
                                                                             throws IOException

Throws:

IOException

createCentroidSupplier

public abstract CentroidSupplier createCentroidSupplier(org.apache.lucene.store.IndexInput centroidsInput,
 int numCentroids,
 org.apache.lucene.index.FieldInfo fieldInfo,
 float[] globalCentroid)
                                                 throws IOException

Throws:

IOException

createCentroidSupplier

public abstract CentroidSupplier createCentroidSupplier(org.apache.lucene.index.FieldInfo info,
 float[][] centroids,
 float[] globalCentroid)
                                                 throws IOException

Throws:

IOException

inheritPreconditioner

protected abstract Preconditioner inheritPreconditioner(org.apache.lucene.index.FieldInfo fieldInfo,
 org.apache.lucene.index.MergeState mergeState)
                                                 throws IOException

Throws:

IOException

createPreconditioner

protected abstract Preconditioner createPreconditioner(int dimension)

writePreconditioner

protected abstract void writePreconditioner(Preconditioner precondtioner,
 org.apache.lucene.store.IndexOutput out)
                                     throws IOException

Throws:

IOException

preconditionVectors

protected abstract org.apache.lucene.index.FloatVectorValues preconditionVectors(Preconditioner precondtioner,
 org.apache.lucene.index.FloatVectorValues vectors)

preconditionVectors

protected abstract Consumer<List<float[]>> preconditionVectors(Preconditioner preconditioner)

flush

public final void flush(int maxDoc,
 org.apache.lucene.index.Sorter.DocMap sortMap)
                 throws IOException

Specified by:

flush in class org.apache.lucene.codecs.KnnVectorsWriter

Throws:

IOException

mergeOneField

public final void mergeOneField(org.apache.lucene.index.FieldInfo fieldInfo,
 org.apache.lucene.index.MergeState mergeState)
                         throws IOException

Overrides:

mergeOneField in class org.apache.lucene.codecs.KnnVectorsWriter

Throws:

IOException

doWriteMeta

protected abstract void doWriteMeta(org.apache.lucene.store.IndexOutput metaOutput,
 org.apache.lucene.index.FieldInfo field,
 int numCentroids,
 long preconditionerOffset,
 long preconditionerLength)
                             throws IOException

Throws:

IOException

finish

public final void finish()
                  throws IOException

Specified by:

finish in class org.apache.lucene.codecs.KnnVectorsWriter

Throws:

IOException

close

public final void close()
                 throws IOException

Throws:

IOException

ramBytesUsed

public final long ramBytesUsed()