Improved examples [skip ci]

Author: ankane

examples/cohere/example.php

@@ -9,7 +9,7 @@
 pg_query($db, 'CREATE TABLE documents (id bigserial PRIMARY KEY, content text, embedding bit(1024))');
 
 // https://docs.cohere.com/reference/embed
-function fetchEmbeddings($texts, $inputType)
+function embed($texts, $inputType)
 {
     $apiKey = getenv('CO_API_KEY') or die("Set CO_API_KEY\n");
     $url = 'https://api.cohere.com/v1/embed';
@@ -36,13 +36,13 @@ function fetchEmbeddings($texts, $inputType)
   'The cat is purring',
   'The bear is growling'
 ];
-$embeddings = fetchEmbeddings($input, 'search_document');
+$embeddings = embed($input, 'search_document');
 foreach ($input as $i => $content) {
     pg_query_params($db, 'INSERT INTO documents (content, embedding) VALUES ($1, $2)', [$content, $embeddings[$i]]);
 }
 
 $query = 'forest';
-$queryEmbedding = fetchEmbeddings([$query], 'search_query')[0];
+$queryEmbedding = embed([$query], 'search_query')[0];
 $result = pg_query_params($db, 'SELECT * FROM documents ORDER BY embedding <~> $1 LIMIT 5', [$queryEmbedding]);
 while ($row = pg_fetch_array($result)) {
     echo $row['content'] . "\n";

examples/hybrid/example.php

@@ -11,8 +11,12 @@
 pg_query($db, 'CREATE TABLE documents (id bigserial PRIMARY KEY, content text, embedding vector(768))');
 pg_query($db, "CREATE INDEX ON documents USING GIN (to_tsvector('english', content))");
 
-function fetchEmbeddings($input)
+function embed($input, $taskType)
 {
+    // nomic-embed-text uses a task prefix
+    // https://huggingface.co/nomic-ai/nomic-embed-text-v1.5
+    $input = array_map(fn($v) => $taskType . ': ' . $v, $input);
+
     $url = 'http://localhost:11434/api/embed';
     $data = [
         'input' => $input,
@@ -35,7 +39,7 @@ function fetchEmbeddings($input)
   'The cat is purring',
   'The bear is growling'
 ];
-$embeddings = fetchEmbeddings($input);
+$embeddings = embed($input, 'search_document');
 
 foreach ($input as $i => $content) {
     pg_query_params($db, 'INSERT INTO documents (content, embedding) VALUES ($1, $2)', [$content, new Vector($embeddings[$i])]);
@@ -65,7 +69,7 @@ function fetchEmbeddings($input)
 LIMIT 5
 SQL;
 $query = 'growling bear';
-$queryEmbedding = fetchEmbeddings($query)[0];
+$queryEmbedding = embed([$query], 'search_query')[0];
 $k = 60;
 $result = pg_query_params($db, $sql, [$query, new Vector($queryEmbedding), $k]);
 while ($row = pg_fetch_array($result)) {

examples/openai/example.php

@@ -10,7 +10,7 @@
 pg_query($db, 'DROP TABLE IF EXISTS documents');
 pg_query($db, 'CREATE TABLE documents (id bigserial PRIMARY KEY, content text, embedding vector(1536))');
 
-function fetchEmbeddings($input)
+function embed($input)
 {
     $apiKey = getenv('OPENAI_API_KEY') or die("Set OPENAI_API_KEY\n");
     $url = 'https://api.openai.com/v1/embeddings';
@@ -35,14 +35,14 @@ function fetchEmbeddings($input)
   'The cat is purring',
   'The bear is growling'
 ];
-$embeddings = fetchEmbeddings($input);
-
+$embeddings = embed($input);
 foreach ($input as $i => $content) {
     pg_query_params($db, 'INSERT INTO documents (content, embedding) VALUES ($1, $2)', [$content, new Vector($embeddings[$i])]);
 }
 
-$documentId = 2;
-$result = pg_query_params($db, 'SELECT * FROM documents WHERE id != $1 ORDER BY embedding <=> (SELECT embedding FROM documents WHERE id = $1) LIMIT 5', [$documentId]);
+$query = 'forest';
+$queryEmbedding = embed([$query])[0];
+$result = pg_query_params($db, 'SELECT * FROM documents ORDER BY embedding <=> $1 LIMIT 5', [new Vector($queryEmbedding)]);
 while ($row = pg_fetch_array($result)) {
     echo $row['content'] . "\n";
 }

examples/sparse/example.php

@@ -17,7 +17,7 @@
 pg_query($db, 'DROP TABLE IF EXISTS documents');
 pg_query($db, 'CREATE TABLE documents (id bigserial PRIMARY KEY, content text, embedding sparsevec(30522))');
 
-function fetchEmbeddings($inputs)
+function embed($inputs)
 {
     $url = 'http://localhost:3000/embed_sparse';
     $data = [
@@ -48,14 +48,13 @@ function fetchEmbeddings($inputs)
   'The cat is purring',
   'The bear is growling'
 ];
-$embeddings = fetchEmbeddings($input);
-
+$embeddings = embed($input);
 foreach ($input as $i => $content) {
     pg_query_params($db, 'INSERT INTO documents (content, embedding) VALUES ($1, $2)', [$content, new SparseVector($embeddings[$i], 30522)]);
 }
 
 $query = 'forest';
-$queryEmbedding = fetchEmbeddings([$query])[0];
+$queryEmbedding = embed([$query])[0];
 $result = pg_query_params($db, 'SELECT content FROM documents ORDER BY embedding <#> $1 LIMIT 5', [new SparseVector($queryEmbedding, 30522)]);
 while ($row = pg_fetch_array($result)) {
     echo $row['content'] . "\n";