Skip to content

Commit f576b17

Browse files
feodorfeodor
committed
Add word_similarity to pg_trgm contrib module.
Patch introduces a concept of similarity over string and just a word from another string. Version of extension is not changed because 1.2 was already introduced in 9.6 release cycle, so, there wasn't a public version. Author: Alexander Korotkov, Artur Zakirov
1 parent 1c4f001 commit f576b17

File tree

10 files changed

+726
-75
lines changed

10 files changed

+726
-75
lines changed

‎contrib/pg_trgm/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ EXTENSION = pg_trgm
77
DATA = pg_trgm--1.2.sql pg_trgm--1.0--1.1.sql pg_trgm--1.1--1.2.sql pg_trgm--unpackaged--1.0.sql
88
PGFILEDESC = "pg_trgm - trigram matching"
99

10-
REGRESS = pg_trgm
10+
REGRESS = pg_trgm pg_word_trgm
1111

1212
ifdef USE_PGXS
1313
PG_CONFIG = pg_config

‎contrib/pg_trgm/expected/pg_trgm.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ select similarity('---', '####---');
5959
0
6060
(1 row)
6161

62-
CREATE TABLE test_trgm(t text);
62+
CREATE TABLE test_trgm(t text COLLATE "C");
6363
\copy test_trgm from 'data/trgm.data'
6464
select t,similarity(t,'qwertyu0988') as sml from test_trgm where t % 'qwertyu0988' order by sml desc, t;
6565
t | sml
@@ -3467,7 +3467,7 @@ select t,similarity(t,'gwertyu1988') as sml from test_trgm where t % 'gwertyu198
34673467
qwertyu0988 | 0.333333
34683468
(1 row)
34693469

3470-
create table test2(t text);
3470+
create table test2(t text COLLATE "C");
34713471
insert into test2 values ('abcdef');
34723472
insert into test2 values ('quark');
34733473
insert into test2 values (' z foo bar');

‎contrib/pg_trgm/pg_trgm--1.1--1.2.sql‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,72 @@
33
-- complain if script is sourced in psql, rather than via ALTER EXTENSION
44
\echo Use "ALTER EXTENSION pg_trgm UPDATE TO '1.2'" to load this file. \quit
55

6+
CREATE FUNCTION word_similarity(text,text)
7+
RETURNS float4
8+
AS 'MODULE_PATHNAME'
9+
LANGUAGE C STRICT IMMUTABLE;
10+
11+
CREATE FUNCTION word_similarity_op(text,text)
12+
RETURNS bool
13+
AS 'MODULE_PATHNAME'
14+
LANGUAGE C STRICT STABLE; -- stable because depends on pg_trgm.word_similarity_threshold
15+
16+
CREATE FUNCTION word_similarity_commutator_op(text,text)
17+
RETURNS bool
18+
AS 'MODULE_PATHNAME'
19+
LANGUAGE C STRICT STABLE; -- stable because depends on pg_trgm.word_similarity_threshold
20+
21+
CREATE FUNCTION word_similarity_dist_op(text,text)
22+
RETURNS float4
23+
AS 'MODULE_PATHNAME'
24+
LANGUAGE C STRICT IMMUTABLE;
25+
26+
CREATE FUNCTION word_similarity_dist_commutator_op(text,text)
27+
RETURNS float4
28+
AS 'MODULE_PATHNAME'
29+
LANGUAGE C STRICT IMMUTABLE;
30+
31+
CREATE OPERATOR <% (
32+
LEFTARG = text,
33+
RIGHTARG = text,
34+
PROCEDURE = word_similarity_op,
35+
COMMUTATOR = '%>',
36+
RESTRICT = contsel,
37+
JOIN = contjoinsel
38+
);
39+
40+
CREATE OPERATOR %> (
41+
LEFTARG = text,
42+
RIGHTARG = text,
43+
PROCEDURE = word_similarity_commutator_op,
44+
COMMUTATOR = '<%',
45+
RESTRICT = contsel,
46+
JOIN = contjoinsel
47+
);
48+
49+
CREATE OPERATOR <<-> (
50+
LEFTARG = text,
51+
RIGHTARG = text,
52+
PROCEDURE = word_similarity_dist_op,
53+
COMMUTATOR = '<->>'
54+
);
55+
56+
CREATE OPERATOR <->> (
57+
LEFTARG = text,
58+
RIGHTARG = text,
59+
PROCEDURE = word_similarity_dist_commutator_op,
60+
COMMUTATOR = '<<->'
61+
);
62+
663
CREATE FUNCTION gin_trgm_triconsistent(internal, int2, text, int4, internal, internal, internal)
764
RETURNS "char"
865
AS 'MODULE_PATHNAME'
966
LANGUAGE C IMMUTABLE STRICT;
1067

68+
ALTER OPERATOR FAMILY gist_trgm_ops USING gist ADD
69+
OPERATOR 7 %> (text, text),
70+
OPERATOR 8 <->> (text, text) FOR ORDER BY pg_catalog.float_ops;
71+
1172
ALTER OPERATOR FAMILY gin_trgm_ops USING gin ADD
12-
FUNCTION 6 (text, text) gin_trgm_triconsistent (internal, int2, text, int4, internal, internal, internal);
73+
OPERATOR 7 %> (text, text),
74+
FUNCTION 6 (text, text) gin_trgm_triconsistent (internal, int2, text, int4, internal, internal, internal);

‎contrib/pg_trgm/pg_trgm--1.2.sql‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,39 @@ CREATE OPERATOR % (
3939
JOIN = contjoinsel
4040
);
4141

42+
CREATE FUNCTION word_similarity(text,text)
43+
RETURNS float4
44+
AS 'MODULE_PATHNAME'
45+
LANGUAGE C STRICT IMMUTABLE;
46+
47+
CREATE FUNCTION word_similarity_op(text,text)
48+
RETURNS bool
49+
AS 'MODULE_PATHNAME'
50+
LANGUAGE C STRICT STABLE; -- stable because depends on pg_trgm.word_similarity_threshold
51+
52+
CREATE FUNCTION word_similarity_commutator_op(text,text)
53+
RETURNS bool
54+
AS 'MODULE_PATHNAME'
55+
LANGUAGE C STRICT STABLE; -- stable because depends on pg_trgm.word_similarity_threshold
56+
57+
CREATE OPERATOR <% (
58+
LEFTARG = text,
59+
RIGHTARG = text,
60+
PROCEDURE = word_similarity_op,
61+
COMMUTATOR = '%>',
62+
RESTRICT = contsel,
63+
JOIN = contjoinsel
64+
);
65+
66+
CREATE OPERATOR %> (
67+
LEFTARG = text,
68+
RIGHTARG = text,
69+
PROCEDURE = word_similarity_commutator_op,
70+
COMMUTATOR = '<%',
71+
RESTRICT = contsel,
72+
JOIN = contjoinsel
73+
);
74+
4275
CREATE FUNCTION similarity_dist(text,text)
4376
RETURNS float4
4477
AS 'MODULE_PATHNAME'
@@ -51,6 +84,30 @@ CREATE OPERATOR <-> (
5184
COMMUTATOR = '<->'
5285
);
5386

87+
CREATE FUNCTION word_similarity_dist_op(text,text)
88+
RETURNS float4
89+
AS 'MODULE_PATHNAME'
90+
LANGUAGE C STRICT IMMUTABLE;
91+
92+
CREATE FUNCTION word_similarity_dist_commutator_op(text,text)
93+
RETURNS float4
94+
AS 'MODULE_PATHNAME'
95+
LANGUAGE C STRICT IMMUTABLE;
96+
97+
CREATE OPERATOR <<-> (
98+
LEFTARG = text,
99+
RIGHTARG = text,
100+
PROCEDURE = word_similarity_dist_op,
101+
COMMUTATOR = '<->>'
102+
);
103+
104+
CREATE OPERATOR <->> (
105+
LEFTARG = text,
106+
RIGHTARG = text,
107+
PROCEDURE = word_similarity_dist_commutator_op,
108+
COMMUTATOR = '<<->'
109+
);
110+
54111
-- gist key
55112
CREATE FUNCTION gtrgm_in(cstring)
56113
RETURNS gtrgm
@@ -140,6 +197,12 @@ ALTER OPERATOR FAMILY gist_trgm_ops USING gist ADD
140197
OPERATOR 5 pg_catalog.~ (text, text),
141198
OPERATOR 6 pg_catalog.~* (text, text);
142199

200+
-- Add operators that are new in 9.6 (pg_trgm 1.2).
201+
202+
ALTER OPERATOR FAMILY gist_trgm_ops USING gist ADD
203+
OPERATOR 7 %> (text, text),
204+
OPERATOR 8 <->> (text, text) FOR ORDER BY pg_catalog.float_ops;
205+
143206
-- support functions for gin
144207
CREATE FUNCTION gin_extract_value_trgm(text, internal)
145208
RETURNS internal
@@ -187,4 +250,5 @@ AS 'MODULE_PATHNAME'
187250
LANGUAGE C IMMUTABLE STRICT;
188251

189252
ALTER OPERATOR FAMILY gin_trgm_ops USING gin ADD
253+
OPERATOR 7 %> (text, text),
190254
FUNCTION 6 (text,text) gin_trgm_triconsistent (internal, int2, text, int4, internal, internal, internal);

‎contrib/pg_trgm/sql/pg_trgm.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ select similarity('wow',' WOW ');
1313

1414
select similarity('---', '####---');
1515

16-
CREATE TABLE test_trgm(t text);
16+
CREATE TABLE test_trgm(t text COLLATE "C");
1717

1818
\copy test_trgm from 'data/trgm.data'
1919

@@ -40,7 +40,7 @@ select t,similarity(t,'qwertyu0988') as sml from test_trgm where t % 'qwertyu098
4040
select t,similarity(t,'gwertyu0988') as sml from test_trgm where t % 'gwertyu0988' order by sml desc, t;
4141
select t,similarity(t,'gwertyu1988') as sml from test_trgm where t % 'gwertyu1988' order by sml desc, t;
4242

43-
create table test2(t text);
43+
create table test2(t text COLLATE "C");
4444
insert into test2 values ('abcdef');
4545
insert into test2 values ('quark');
4646
insert into test2 values (' z foo bar');

‎contrib/pg_trgm/trgm.h‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,14 @@
2626
#define DIVUNION
2727

2828
/* operator strategy numbers */
29-
#define SimilarityStrategyNumber 1
30-
#define DistanceStrategyNumber 2
31-
#define LikeStrategyNumber 3
32-
#define ILikeStrategyNumber 4
33-
#define RegExpStrategyNumber 5
34-
#define RegExpICaseStrategyNumber 6
35-
29+
#define SimilarityStrategyNumber 1
30+
#define DistanceStrategyNumber 2
31+
#define LikeStrategyNumber 3
32+
#define ILikeStrategyNumber 4
33+
#define RegExpStrategyNumber 5
34+
#define RegExpICaseStrategyNumber 6
35+
#define WordSimilarityStrategyNumber 7
36+
#define WordDistanceStrategyNumber 8
3637

3738
typedef char trgm[3];
3839

@@ -103,15 +104,28 @@ typedef char *BITVECP;
103104
#define GETARR(x) ( (trgm*)( (char*)x+TRGMHDRSIZE ) )
104105
#define ARRNELEM(x) ( ( VARSIZE(x) - TRGMHDRSIZE )/sizeof(trgm) )
105106

107+
/*
108+
* If DIVUNION is defined then similarity formula is:
109+
* count / (len1 + len2 - count)
110+
* else if DIVUNION is not defined then similarity formula is:
111+
* count / max(len1, len2)
112+
*/
113+
#ifdef DIVUNION
114+
#define CALCSML(count, len1, len2) ((float4) (count)) / ((float4) ((len1) + (len2) - (count)))
115+
#else
116+
#define CALCSML(count, len1, len2) ((float4) (count)) / ((float4) (((len1) > (len2)) ? (len1) : (len2)))
117+
#endif
118+
106119
typedef struct TrgmPackedGraph TrgmPackedGraph;
107120

108121
extern double similarity_threshold;
122+
extern double word_similarity_threshold;
109123

110124
extern uint32 trgm2int(trgm *ptr);
111125
extern void compact_trigram(trgm *tptr, char *str, int bytelen);
112126
extern TRGM *generate_trgm(char *str, int slen);
113127
extern TRGM *generate_wildcard_trgm(const char *str, int slen);
114-
extern float4 cnt_sml(TRGM *trg1, TRGM *trg2);
128+
extern float4 cnt_sml(TRGM *trg1, TRGM *trg2, bool inexact);
115129
extern bool trgm_contained_by(TRGM *trg1, TRGM *trg2);
116130
extern bool *trgm_presence_map(TRGM *query, TRGM *key);
117131
extern TRGM *createTrgmNFA(text *text_re, Oid collation,

‎contrib/pg_trgm/trgm_gin.c‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ gin_extract_query_trgm(PG_FUNCTION_ARGS)
8989
switch (strategy)
9090
{
9191
case SimilarityStrategyNumber:
92+
case WordSimilarityStrategyNumber:
9293
trg = generate_trgm(VARDATA(val), VARSIZE(val) - VARHDRSZ);
9394
break;
9495
case ILikeStrategyNumber:
@@ -176,13 +177,18 @@ gin_trgm_consistent(PG_FUNCTION_ARGS)
176177
bool res;
177178
int32 i,
178179
ntrue;
180+
double nlimit;
179181

180182
/* All cases served by this function are inexact */
181183
*recheck = true;
182184

183185
switch (strategy)
184186
{
185187
case SimilarityStrategyNumber:
188+
case WordSimilarityStrategyNumber:
189+
nlimit = (strategy == SimilarityStrategyNumber) ?
190+
similarity_threshold : word_similarity_threshold;
191+
186192
/* Count the matches */
187193
ntrue = 0;
188194
for (i = 0; i < nkeys; i++)
@@ -207,8 +213,7 @@ gin_trgm_consistent(PG_FUNCTION_ARGS)
207213
* So, independly on DIVUNION the upper bound formula is the same.
208214
*/
209215
res = (nkeys == 0) ? false :
210-
((((((float4) ntrue) / ((float4) nkeys))) >= similarity_threshold)
211-
? true : false);
216+
(((((float4) ntrue) / ((float4) nkeys))) >= nlimit);
212217
break;
213218
case ILikeStrategyNumber:
214219
#ifndef IGNORECASE
@@ -270,10 +275,15 @@ gin_trgm_triconsistent(PG_FUNCTION_ARGS)
270275
int32 i,
271276
ntrue;
272277
bool *boolcheck;
278+
double nlimit;
273279

274280
switch (strategy)
275281
{
276282
case SimilarityStrategyNumber:
283+
case WordSimilarityStrategyNumber:
284+
nlimit = (strategy == SimilarityStrategyNumber) ?
285+
similarity_threshold : word_similarity_threshold;
286+
277287
/* Count the matches */
278288
ntrue = 0;
279289
for (i = 0; i < nkeys; i++)
@@ -285,9 +295,9 @@ gin_trgm_triconsistent(PG_FUNCTION_ARGS)
285295
/*
286296
* See comment in gin_trgm_consistent() about * upper bound formula
287297
*/
288-
res = (nkeys == 0) ? GIN_FALSE :
289-
(((((float4) ntrue) / ((float4) nkeys)) >= similarity_threshold)
290-
? GIN_MAYBE : GIN_FALSE);
298+
res = (nkeys == 0)
299+
? GIN_FALSE : (((((float4) ntrue) / ((float4) nkeys)) >= nlimit)
300+
? GIN_MAYBE : GIN_FALSE);
291301
break;
292302
case ILikeStrategyNumber:
293303
#ifndef IGNORECASE

0 commit comments

Comments
 (0)