Skip to content

Commit e47ef81

Browse files
ceviancevian
committed
allow upserts to take metadata as dict
1 parent 426794a commit e47ef81

5 files changed

Lines changed: 121 additions & 14 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ This takes three arguments:
3838
In this tutorial, we will use the async client. But we have a sync
3939
client as well (with an almost identical interface)
4040

41+
``` python
42+
from timescale_vector import client
43+
```
44+
4145
``` python
4246
vec = client.Async(connection_string, "my_data", 2)
4347
```

‎nbs/00_vector.ipynb‎

Lines changed: 88 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,8 @@
243243
"text/markdown": [
244244
"---\n",
245245
"\n",
246+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L79){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
247+
"\n",
246248
"### QueryBuilder.get_create_query\n",
247249
"\n",
248250
"> QueryBuilder.get_create_query ()\n",
@@ -255,6 +257,8 @@
255257
"text/plain": [
256258
"---\n",
257259
"\n",
260+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L79){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
261+
"\n",
258262
"### QueryBuilder.get_create_query\n",
259263
"\n",
260264
"> QueryBuilder.get_create_query ()\n",
@@ -333,6 +337,12 @@
333337
" rec = await pool.fetchrow(query)\n",
334338
" return rec == None\n",
335339
"\n",
340+
" def _convert_record_meta_to_json(item):\n",
341+
" if not isinstance(item[1], dict):\n",
342+
" raise ValueError(\"Cannot mix dictionary and string metadata fields in the same upsert\")\n",
343+
" return (item[0], json.dumps(item[1]), item[2], item[3])\n",
344+
"\n",
345+
"\n",
336346
" async def upsert(self, records):\n",
337347
" \"\"\"\n",
338348
" Performs upsert operation for multiple records.\n",
@@ -343,6 +353,8 @@
343353
" Returns:\n",
344354
" None\n",
345355
" \"\"\"\n",
356+
" if isinstance(records[0][1], dict):\n",
357+
" records = list(map(lambda item: Async._convert_record_meta_to_json(item), records))\n",
346358
" query = self.builder.get_upsert_query()\n",
347359
" async with await self.connect() as pool:\n",
348360
" await pool.executemany(query, records)\n",
@@ -422,6 +434,8 @@
422434
"text/markdown": [
423435
"---\n",
424436
"\n",
437+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L229){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
438+
"\n",
425439
"### Async.create_tables\n",
426440
"\n",
427441
"> Async.create_tables ()\n",
@@ -434,6 +448,8 @@
434448
"text/plain": [
435449
"---\n",
436450
"\n",
451+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L229){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
452+
"\n",
437453
"### Async.create_tables\n",
438454
"\n",
439455
"> Async.create_tables ()\n",
@@ -463,6 +479,8 @@
463479
"text/markdown": [
464480
"---\n",
465481
"\n",
482+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L229){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
483+
"\n",
466484
"### Async.create_tables\n",
467485
"\n",
468486
"> Async.create_tables ()\n",
@@ -475,6 +493,8 @@
475493
"text/plain": [
476494
"---\n",
477495
"\n",
496+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L229){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
497+
"\n",
478498
"### Async.create_tables\n",
479499
"\n",
480500
"> Async.create_tables ()\n",
@@ -504,6 +524,8 @@
504524
"text/markdown": [
505525
"---\n",
506526
"\n",
527+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L279){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
528+
"\n",
507529
"### Async.search\n",
508530
"\n",
509531
"> Async.search (query_embedding:List[float], k:int=10,\n",
@@ -517,6 +539,8 @@
517539
"text/plain": [
518540
"---\n",
519541
"\n",
542+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L279){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
543+
"\n",
520544
"### Async.search\n",
521545
"\n",
522546
"> Async.search (query_embedding:List[float], k:int=10,\n",
@@ -566,7 +590,7 @@
566590
"await vec.create_tables()\n",
567591
"empty = await vec.table_is_empty()\n",
568592
"assert empty\n",
569-
"await vec.upsert([(uuid.uuid4(), '''{\"key\":\"val\"}''', \"the brown fox\", [1.0,1.2])])\n",
593+
"await vec.upsert([(uuid.uuid4(), {\"key\" : \"val\"}, \"the brown fox\", [1.0,1.2])])\n",
570594
"empty = await vec.table_is_empty()\n",
571595
"assert not empty\n",
572596
"\n",
@@ -603,7 +627,27 @@
603627
"rec = await vec.search([1.0, 2.0], filter={\"key_1\":\"val_1\", \"key_2\":\"val_2\"})\n",
604628
"assert len(rec) == 1\n",
605629
"rec = await vec.search([1.0, 2.0], k=4, filter={\"key_1\":\"val_1\", \"key_2\":\"val_3\"})\n",
606-
"assert len(rec) == 0"
630+
"assert len(rec) == 0\n",
631+
"\n",
632+
"try:\n",
633+
" # can't upsert using both keys and dictionaries\n",
634+
" await vec.upsert([ \\\n",
635+
" (uuid.uuid4(), {\"key\" : \"val\"}, \"the brown fox\", [1.0,1.2]), \\\n",
636+
" (uuid.uuid4(), '''{\"key2\":\"val\"}''' , \"the brown fox\", [1.0,1.2])\\\n",
637+
" ])\n",
638+
" assert False\n",
639+
"except ValueError as e:\n",
640+
" pass\n",
641+
"\n",
642+
"try:\n",
643+
" # can't upsert using both keys and dictionaries opposite order\n",
644+
" await vec.upsert([ \\\n",
645+
" (uuid.uuid4(), '''{\"key2\":\"val\"}''', \"the brown fox\", [1.0,1.2]), \\\n",
646+
" (uuid.uuid4(), {\"key\" : \"val\"}, \"the brown fox\", [1.0,1.2])\\\n",
647+
" ])\n",
648+
" assert False\n",
649+
"except BaseException as e:\n",
650+
" pass\n"
607651
]
608652
},
609653
{
@@ -715,6 +759,11 @@
715759
" rec = cur.fetchone()\n",
716760
" return rec == None\n",
717761
"\n",
762+
" def _convert_record_meta_to_json(item):\n",
763+
" if not isinstance(item[1], dict):\n",
764+
" raise ValueError(\"Cannot mix dictionary and string metadata fields in the same upsert\")\n",
765+
" return (item[0], json.dumps(item[1]), item[2], item[3])\n",
766+
" \n",
718767
" def upsert(self, records):\n",
719768
" \"\"\"\n",
720769
" Performs upsert operation for multiple records.\n",
@@ -725,6 +774,9 @@
725774
" Returns:\n",
726775
" None\n",
727776
" \"\"\"\n",
777+
" if isinstance(records[0][1], dict):\n",
778+
" records = list(map(lambda item: Async._convert_record_meta_to_json(item), records))\n",
779+
" \n",
728780
" query = self.builder.get_upsert_query()\n",
729781
" query, _ = self._translate_to_pyformat(query, None)\n",
730782
" with self.connect() as conn:\n",
@@ -816,6 +868,8 @@
816868
"text/markdown": [
817869
"---\n",
818870
"\n",
871+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L398){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
872+
"\n",
819873
"### Sync.create_tables\n",
820874
"\n",
821875
"> Sync.create_tables ()\n",
@@ -828,6 +882,8 @@
828882
"text/plain": [
829883
"---\n",
830884
"\n",
885+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L398){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
886+
"\n",
831887
"### Sync.create_tables\n",
832888
"\n",
833889
"> Sync.create_tables ()\n",
@@ -857,6 +913,8 @@
857913
"text/markdown": [
858914
"---\n",
859915
"\n",
916+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L382){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
917+
"\n",
860918
"### Sync.upsert\n",
861919
"\n",
862920
"> Sync.upsert (records)\n",
@@ -872,6 +930,8 @@
872930
"text/plain": [
873931
"---\n",
874932
"\n",
933+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L382){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
934+
"\n",
875935
"### Sync.upsert\n",
876936
"\n",
877937
"> Sync.upsert (records)\n",
@@ -904,6 +964,8 @@
904964
"text/markdown": [
905965
"---\n",
906966
"\n",
967+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L453){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
968+
"\n",
907969
"### Sync.search\n",
908970
"\n",
909971
"> Sync.search (query_embedding:List[float], k:int=10,\n",
@@ -922,6 +984,8 @@
922984
"text/plain": [
923985
"---\n",
924986
"\n",
987+
"[source](https://github.com/timescale/python-vector/blob/main/timescale_vector/client.py#L453){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
988+
"\n",
925989
"### Sync.search\n",
926990
"\n",
927991
"> Sync.search (query_embedding:List[float], k:int=10,\n",
@@ -977,7 +1041,7 @@
9771041
"empty = vec.table_is_empty()\n",
9781042
"\n",
9791043
"assert empty\n",
980-
"vec.upsert([(uuid.uuid4(), '''{\"key\":\"val\"}''', \"the brown fox\", [1.0,1.2])])\n",
1044+
"vec.upsert([(uuid.uuid4(), {\"key\" : \"val\"}, \"the brown fox\", [1.0,1.2])])\n",
9811045
"empty = vec.table_is_empty()\n",
9821046
"assert not empty\n",
9831047
"\n",
@@ -1016,7 +1080,27 @@
10161080
"rec = vec.search([1.0, 2.0], filter={\"key_1\":\"val_1\", \"key_2\":\"val_2\"})\n",
10171081
"assert len(rec) == 1\n",
10181082
"rec = vec.search([1.0, 2.0], k=4, filter={\"key_1\":\"val_1\", \"key_2\":\"val_3\"})\n",
1019-
"assert len(rec) == 0"
1083+
"assert len(rec) == 0\n",
1084+
"\n",
1085+
"try:\n",
1086+
" # can't upsert using both keys and dictionaries\n",
1087+
" await vec.upsert([ \\\n",
1088+
" (uuid.uuid4(), {\"key\" : \"val\"}, \"the brown fox\", [1.0,1.2]), \\\n",
1089+
" (uuid.uuid4(), '''{\"key2\":\"val\"}''' , \"the brown fox\", [1.0,1.2])\\\n",
1090+
" ])\n",
1091+
" assert False\n",
1092+
"except ValueError as e:\n",
1093+
" pass\n",
1094+
"\n",
1095+
"try:\n",
1096+
" # can't upsert using both keys and dictionaries opposite order\n",
1097+
" await vec.upsert([ \\\n",
1098+
" (uuid.uuid4(), '''{\"key2\":\"val\"}''', \"the brown fox\", [1.0,1.2]), \\\n",
1099+
" (uuid.uuid4(), {\"key\" : \"val\"}, \"the brown fox\", [1.0,1.2])\\\n",
1100+
" ])\n",
1101+
" assert False\n",
1102+
"except BaseException as e:\n",
1103+
" pass"
10201104
]
10211105
},
10221106
{

‎nbs/index.ipynb‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,5 @@
11
{
22
"cells": [
3-
{
4-
"cell_type": "code",
5-
"execution_count": null,
6-
"metadata": {},
7-
"outputs": [],
8-
"source": [
9-
"#| hide\n",
10-
"from timescale_vector import client "
11-
]
12-
},
133
{
144
"cell_type": "markdown",
155
"metadata": {},
@@ -113,6 +103,15 @@
113103
"await con.close()"
114104
]
115105
},
106+
{
107+
"cell_type": "code",
108+
"execution_count": null,
109+
"metadata": {},
110+
"outputs": [],
111+
"source": [
112+
"from timescale_vector import client "
113+
]
114+
},
116115
{
117116
"cell_type": "code",
118117
"execution_count": null,

‎timescale_vector/_modidx.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
'syms': { 'timescale_vector.client': { 'timescale_vector.client.Async': ('vector.html#async', 'timescale_vector/client.py'),
99
'timescale_vector.client.Async.__init__': ( 'vector.html#async.__init__',
1010
'timescale_vector/client.py'),
11+
'timescale_vector.client.Async._convert_record_meta_to_json': ( 'vector.html#async._convert_record_meta_to_json',
12+
'timescale_vector/client.py'),
1113
'timescale_vector.client.Async._get_approx_count': ( 'vector.html#async._get_approx_count',
1214
'timescale_vector/client.py'),
1315
'timescale_vector.client.Async.connect': ( 'vector.html#async.connect',
@@ -46,6 +48,8 @@
4648
'timescale_vector.client.Sync': ('vector.html#sync', 'timescale_vector/client.py'),
4749
'timescale_vector.client.Sync.__init__': ( 'vector.html#sync.__init__',
4850
'timescale_vector/client.py'),
51+
'timescale_vector.client.Sync._convert_record_meta_to_json': ( 'vector.html#sync._convert_record_meta_to_json',
52+
'timescale_vector/client.py'),
4953
'timescale_vector.client.Sync._get_approx_count': ( 'vector.html#sync._get_approx_count',
5054
'timescale_vector/client.py'),
5155
'timescale_vector.client.Sync._translate_to_pyformat': ( 'vector.html#sync._translate_to_pyformat',

‎timescale_vector/client.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,12 @@ async def table_is_empty(self):
212212
rec = await pool.fetchrow(query)
213213
return rec == None
214214

215+
def _convert_record_meta_to_json(item):
216+
if not isinstance(item[1], dict):
217+
raise ValueError("Cannot mix dictionary and string metadata fields in the same upsert")
218+
return (item[0], json.dumps(item[1]), item[2], item[3])
219+
220+
215221
async def upsert(self, records):
216222
"""
217223
Performs upsert operation for multiple records.
@@ -222,6 +228,8 @@ async def upsert(self, records):
222228
Returns:
223229
None
224230
"""
231+
if isinstance(records[0][1], dict):
232+
records = list(map(lambda item: Async._convert_record_meta_to_json(item), records))
225233
query = self.builder.get_upsert_query()
226234
async with await self.connect() as pool:
227235
await pool.executemany(query, records)
@@ -379,6 +387,11 @@ def table_is_empty(self):
379387
rec = cur.fetchone()
380388
return rec == None
381389

390+
def _convert_record_meta_to_json(item):
391+
if not isinstance(item[1], dict):
392+
raise ValueError("Cannot mix dictionary and string metadata fields in the same upsert")
393+
return (item[0], json.dumps(item[1]), item[2], item[3])
394+
382395
def upsert(self, records):
383396
"""
384397
Performs upsert operation for multiple records.
@@ -389,6 +402,9 @@ def upsert(self, records):
389402
Returns:
390403
None
391404
"""
405+
if isinstance(records[0][1], dict):
406+
records = list(map(lambda item: Async._convert_record_meta_to_json(item), records))
407+
392408
query = self.builder.get_upsert_query()
393409
query, _ = self._translate_to_pyformat(query, None)
394410
with self.connect() as conn:

0 commit comments

Comments
 (0)