Skip to content

Instantly share code, notes, and snippets.

@mingodad
Created September 24, 2018 13:43
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save mingodad/2f05cd1280d58f93f89133b2a2011a4d to your computer and use it in GitHub Desktop.
Save mingodad/2f05cd1280d58f93f89133b2a2011a4d to your computer and use it in GitHub Desktop.
Here is the diffs to an implementation of "notcompressed" option for columns in fts3/4 in sqlite3
Index: ext/fts3/fts3.c
==================================================================
--- ext/fts3/fts3.c
+++ ext/fts3/fts3.c
@@ -851,11 +851,12 @@
}else{
zFree = zFunction = fts3QuoteId(zFunc);
}
fts3Appendf(pRc, &zRet, "docid");
for(i=0; i<p->nColumn; i++){
- fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
+ fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')",
+ p->abNotcompressed[i] ? "" : zFunction, i, p->azColumn[i]);
}
if( p->zLanguageid ){
fts3Appendf(pRc, &zRet, ", x.%Q", "langid");
}
sqlite3_free(zFree);
@@ -907,11 +908,11 @@
}else{
zFree = zFunction = fts3QuoteId(zFunc);
}
fts3Appendf(pRc, &zRet, "?");
for(i=0; i<p->nColumn; i++){
- fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
+ fts3Appendf(pRc, &zRet, ",%s(?)", p->abNotcompressed[i] ? "" : zFunction);
}
if( p->zLanguageid ){
fts3Appendf(pRc, &zRet, ", ?");
}
sqlite3_free(zFree);
@@ -1145,10 +1146,12 @@
char *zUncompress = 0; /* uncompress=? parameter (or NULL) */
char *zContent = 0; /* content=? parameter (or NULL) */
char *zLanguageid = 0; /* languageid=? parameter (or NULL) */
char **azNotindexed = 0; /* The set of notindexed= columns */
int nNotindexed = 0; /* Size of azNotindexed[] array */
+ char **azNotcompressed = 0; /* The set of notcompessed= columns */
+ int nNotcompressed = 0; /* Size of azNotindexed[] array */
assert( strlen(argv[0])==4 );
assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
|| (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
);
@@ -1159,15 +1162,19 @@
nByte = sizeof(const char *) * (argc-2);
aCol = (const char **)sqlite3_malloc(nByte);
if( aCol ){
memset((void*)aCol, 0, nByte);
azNotindexed = (char **)sqlite3_malloc(nByte);
+ azNotcompressed = (char **)sqlite3_malloc(nByte);
}
if( azNotindexed ){
memset(azNotindexed, 0, nByte);
}
- if( !aCol || !azNotindexed ){
+ if( azNotcompressed ){
+ memset(azNotcompressed, 0, nByte);
+ }
+ if( !aCol || !azNotindexed | !azNotcompressed){
rc = SQLITE_NOMEM;
goto fts3_init_out;
}
/* Loop through all of the arguments passed by the user to the FTS3/4
@@ -1206,11 +1213,12 @@
{ "compress", 8 }, /* 2 -> COMPRESS */
{ "uncompress", 10 }, /* 3 -> UNCOMPRESS */
{ "order", 5 }, /* 4 -> ORDER */
{ "content", 7 }, /* 5 -> CONTENT */
{ "languageid", 10 }, /* 6 -> LANGUAGEID */
- { "notindexed", 10 } /* 7 -> NOTINDEXED */
+ { "notindexed", 10 }, /* 7 -> NOTINDEXED */
+ { "notcompressed", 13 } /* 7 -> NOTCOMPRESSED */
};
int iOpt;
if( !zVal ){
rc = SQLITE_NOMEM;
@@ -1273,10 +1281,15 @@
case 7: /* NOTINDEXED */
azNotindexed[nNotindexed++] = zVal;
zVal = 0;
break;
+
+ case 8: /* NOTCOMPRESSED */
+ azNotcompressed[nNotcompressed++] = zVal;
+ zVal = 0;
+ break;
default:
assert( iOpt==SizeofArray(aFts4Opt) );
sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z);
rc = SQLITE_ERROR;
@@ -1350,10 +1363,11 @@
/* Allocate and populate the Fts3Table structure. */
nByte = sizeof(Fts3Table) + /* Fts3Table */
nCol * sizeof(char *) + /* azColumn */
nIndex * sizeof(struct Fts3Index) + /* aIndex */
nCol * sizeof(u8) + /* abNotindexed */
+ nCol * sizeof(u8) + /* abNotcompressed */
nName + /* zName */
nDb + /* zDb */
nString; /* Space for azColumn strings */
p = (Fts3Table*)sqlite3_malloc(nByte);
if( p==0 ){
@@ -1384,13 +1398,14 @@
p->nIndex = nIndex;
for(i=0; i<nIndex; i++){
fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1);
}
p->abNotindexed = (u8 *)&p->aIndex[nIndex];
+ p->abNotcompressed = (u8 *)&p->abNotindexed[nCol];
/* Fill in the zName and zDb fields of the vtab structure. */
- zCsr = (char *)&p->abNotindexed[nCol];
+ zCsr = (char *)&p->abNotcompressed[nCol];
p->zName = zCsr;
memcpy(zCsr, argv[2], nName);
zCsr += nName;
p->zDb = zCsr;
memcpy(zCsr, argv[1], nDb);
@@ -1429,10 +1444,31 @@
if( azNotindexed[i] ){
sqlite3Fts3ErrMsg(pzErr, "no such column: %s", azNotindexed[i]);
rc = SQLITE_ERROR;
}
}
+
+ /* Fill in the abNotcompressed array */
+ for(iCol=0; iCol<nCol; iCol++){
+ int n = (int)strlen(p->azColumn[iCol]);
+ for(i=0; i<nNotcompressed; i++){
+ char *zNot = azNotcompressed[i];
+ if( zNot && n==(int)strlen(zNot)
+ && 0==sqlite3_strnicmp(p->azColumn[iCol], zNot, n)
+ ){
+ p->abNotcompressed[iCol] = 1;
+ sqlite3_free(zNot);
+ azNotcompressed[i] = 0;
+ }
+ }
+ }
+ for(i=0; i<nNotcompressed; i++){
+ if( azNotcompressed[i] ){
+ sqlite3Fts3ErrMsg(pzErr, "no such column: %s", azNotcompressed[i]);
+ rc = SQLITE_ERROR;
+ }
+ }
if( rc==SQLITE_OK && (zCompress==0)!=(zUncompress==0) ){
char const *zMiss = (zCompress==0 ? "compress" : "uncompress");
rc = SQLITE_ERROR;
sqlite3Fts3ErrMsg(pzErr, "missing %s parameter in fts4 constructor", zMiss);
@@ -1469,12 +1505,14 @@
sqlite3_free(zCompress);
sqlite3_free(zUncompress);
sqlite3_free(zContent);
sqlite3_free(zLanguageid);
for(i=0; i<nNotindexed; i++) sqlite3_free(azNotindexed[i]);
+ for(i=0; i<nNotcompressed; i++) sqlite3_free(azNotcompressed[i]);
sqlite3_free((void *)aCol);
sqlite3_free((void *)azNotindexed);
+ sqlite3_free((void *)azNotcompressed);
if( rc!=SQLITE_OK ){
if( p ){
fts3DisconnectMethod((sqlite3_vtab *)p);
}else if( pTokenizer ){
pTokenizer->pModule->xDestroy(pTokenizer);
@@ -3899,10 +3937,11 @@
void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule);
#endif
#ifdef SQLITE_ENABLE_ICU
void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
#endif
+void sqlite3Fts3SubLatinTokenizerModule(sqlite3_tokenizer_module const**ppModule);
Index: ext/fts3/fts3Int.h
==================================================================
--- ext/fts3/fts3Int.h
+++ ext/fts3/fts3Int.h
@@ -218,10 +218,11 @@
const char *zDb; /* logical database name */
const char *zName; /* virtual table name */
int nColumn; /* number of named columns in virtual table */
char **azColumn; /* column names. malloced */
u8 *abNotindexed; /* True for 'notindexed' columns */
+ u8 *abNotcompressed; /* True for 'notcompressed' columns */
sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
char *zContentTbl; /* content=xxx option, or NULL */
char *zLanguageid; /* languageid=xxx option, or NULL */
int nAutoincrmerge; /* Value configured by 'automerge' */
u32 nLeafAdd; /* Number of leaf blocks added this trans */
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment