Skip to content

Commit b393254

Browse files
calccryptocalccrypto
committed
Updates and fixes
GUFI-tree -> GUFI index addqueryfuncs has path, fpath, and epath again added thread id argument to the function if there are no threads, use a consistent value in [0, MAXPTHREAD) Updated gufi_query -S - might need to change callback function Fixed querydbn
1 parent 5e326e4 commit b393254

10 files changed

+83
-55
lines changed

‎LICENSE.txt

-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
/*
21
This file is part of GUFI, which is part of MarFS, which is released
32
under the BSD license.
43

@@ -58,4 +57,3 @@ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
5857
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
5958
IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
6059
OF SUCH DAMAGE.
61-
*/

‎include/dbutils.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -139,10 +139,10 @@ int insertsumdb(sqlite3 *sdb, struct work *pwork,struct sum *su);
139139

140140
int inserttreesumdb(const char *name, sqlite3 *sdb, struct sum *su,int rectype,int uid,int gid);
141141

142-
int addqueryfuncs(sqlite3 *db);
142+
int addqueryfuncs(sqlite3 *db, int id);
143143

144144
size_t print_results(sqlite3_stmt *res, FILE *out, const int printpath, const int printheader, const int printrows, const char *delim);
145145

146-
sqlite3 *open_aggregate(const char *name, const char *attach_name, const char *query);
146+
sqlite3 *open_aggregate(const char *name, const char *attach_name, const char *query, int id);
147147

148148
#endif

‎src/bf.c

+4-4
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ void print_help(const char* prog_name,
9595
if (! opt)
9696
return;
9797

98-
printf("Usage: %s [options] %s\n", prog_name, positional_args_help_str);
98+
printf("usage: %s [options] %s\n", prog_name, positional_args_help_str);
9999
printf("options:\n");
100100

101101
int ch;
@@ -115,7 +115,7 @@ void print_help(const char* prog_name,
115115
case 'n': printf(" -n <threads> number of threads\n"); break;
116116
case 'd': printf(" -d <delim> delimiter (one char) [use 'x' for 0x%02X]\n", (uint8_t)fielddelim[0]); break;
117117
case 'i': printf(" -i <input_dir> input directory path\n"); break;
118-
case 't': printf(" -t <to_dir> build GUFI-tree (under) here\n"); break;
118+
case 't': printf(" -t <to_dir> build GUFI index (under) here\n"); break;
119119
case 'o': printf(" -o <out_fname> output file (one-per-thread, with thread-id suffix), implies -e 1\n"); break;
120120
case 'O': printf(" -O <out_DB> output DB, implies -e 1\n"); break;
121121
case 'I': printf(" -I <SQL_init> SQL init\n"); break;
@@ -135,8 +135,8 @@ void print_help(const char* prog_name,
135135
case 'u': printf(" -u input mode is from a file so input is a file not a dir\n"); break;
136136
case 'y': printf(" -y <min level> minimum level to go down\n"); break;
137137
case 'z': printf(" -z <max level> maximum level to go down\n"); break;
138-
case 'G': printf(" -G <SQL_aggregate> SQL for aggregated results (deaults to \"SELECT * FROM entries\")\n"); break;
139-
case 'J': printf(" -J <SQL_interm> SQL for intermediate results (deaults to \"SELECT * FROM entries\")\n"); break;
138+
case 'G': printf(" -G <SQL_aggregate> SQL for aggregated results (no default: recommend using \"SELECT * FROM entries\")\n"); break;
139+
case 'J': printf(" -J <SQL_interm> SQL for intermediate results (no default: recommend using \"SELECT * FROM entries\")\n"); break;
140140
case 'e': printf(" -e <0 or 1> 0 for aggregate, 1 for print without aggregating (implied by -o and -O)\n"); break;
141141
case 'm': printf(" -m Keep mtime and atime same on the database files\n"); break;
142142
case 'B': printf(" -B <buffer size> size of each thread's output buffer in bytes\n"); break;

‎src/bfti.c

+3-3
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@ int processfin() {
238238

239239

240240
void sub_help() {
241-
printf("GUFI_tree path to GUFI tree-dir\n");
241+
printf("GUFI_index path to GUFI index\n");
242242
printf("\n");
243243
}
244244

@@ -257,15 +257,15 @@ int main(int argc, char *argv[])
257257
// but allow different fields to be filled at the command-line.
258258
// Callers provide the options-string for get_opt(), which will
259259
// control which options are parsed for each program.
260-
int idx = parse_cmd_line(argc, argv, "hHPn:s", 1, "GUFI_tree", &in);
260+
int idx = parse_cmd_line(argc, argv, "hHPn:s", 1, "GUFI_index", &in);
261261
if (in.helped)
262262
sub_help();
263263
if (idx < 0)
264264
return -1;
265265
else {
266266
// parse positional args, following the options
267267
int retval = 0;
268-
INSTALL_STR(in.name, argv[idx++], MAXPATH, "GUFI_tree");
268+
INSTALL_STR(in.name, argv[idx++], MAXPATH, "GUFI_index");
269269

270270
if (retval)
271271
return retval;

‎src/bfwreaddirplus2db.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,7 @@ static int processdir(struct QPTPool * ctx, const size_t id, void * data, void *
318318

319319
SNPRINTF(passmywork->type,2,"%s","d");
320320
passmywork->suspect=in.suspectd;
321-
/* if we are putting the gufi tree into the source tree we can modify the suspecttime to be the mtime of the gufi db */
321+
/* if we are putting the gufi index into the source tree we can modify the suspecttime to be the mtime of the gufi db */
322322
/* this way we will just be looking at dirs or files that have changed since the gufi db was last updated */
323323
locsuspecttime=in.suspecttime;
324324
if (in.buildinindir == 1) {
@@ -716,7 +716,7 @@ int validate_inputs() {
716716
}
717717

718718
void sub_help() {
719-
printf("input_dir walk this tree to produce GUFI-tree\n");
719+
printf("input_dir walk this tree to produce GUFI index\n");
720720
printf("\n");
721721
}
722722

‎src/dbutils.c

+32-7
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,9 @@ OF SUCH DAMAGE.
6262

6363

6464
#include <errno.h>
65-
#include <string.h>
65+
#include <stdint.h>
6666
#include <stdlib.h>
67+
#include <string.h>
6768

6869
#include <pwd.h>
6970
#include <grp.h>
@@ -813,6 +814,27 @@ int inserttreesumdb(const char *name, sqlite3 *sdb, struct sum *su,int rectype,i
813814
return 0;
814815
}
815816

817+
static void path(sqlite3_context *context, int argc, sqlite3_value **argv)
818+
{
819+
const size_t id = (int) (uintptr_t) sqlite3_user_data(context);
820+
sqlite3_result_text(context, gps[id].gpath, -1, SQLITE_TRANSIENT);
821+
return;
822+
}
823+
824+
static void fpath(sqlite3_context *context, int argc, sqlite3_value **argv)
825+
{
826+
const size_t id = (int) (uintptr_t) sqlite3_user_data(context);
827+
sqlite3_result_text(context, gps[id].gfpath, -1, SQLITE_TRANSIENT);
828+
return;
829+
}
830+
831+
static void epath(sqlite3_context *context, int argc, sqlite3_value **argv)
832+
{
833+
const size_t id = (int) (uintptr_t) sqlite3_user_data(context);
834+
sqlite3_result_text(context, gps[id].gepath, -1, SQLITE_TRANSIENT);
835+
return;
836+
}
837+
816838
static void uidtouser(sqlite3_context *context, int argc, sqlite3_value **argv)
817839
{
818840
struct passwd *fmypasswd;
@@ -864,10 +886,13 @@ static void modetotxt(sqlite3_context *context, int argc, sqlite3_value **argv)
864886
sqlite3_result_null(context);
865887
}
866888

867-
int addqueryfuncs(sqlite3 *db) {
868-
return ((sqlite3_create_function(db, "uidtouser", 1, SQLITE_UTF8, NULL, &uidtouser, NULL, NULL) == SQLITE_OK) &&
869-
(sqlite3_create_function(db, "gidtogroup", 1, SQLITE_UTF8, NULL, &gidtogroup, NULL, NULL) == SQLITE_OK) &&
870-
(sqlite3_create_function(db, "modetotxt", 1, SQLITE_UTF8, NULL, &modetotxt, NULL, NULL) == SQLITE_OK))?0:1;
889+
int addqueryfuncs(sqlite3 *db, int id) {
890+
return ((sqlite3_create_function(db, "path", 0, SQLITE_UTF8, (void *) (uintptr_t) id, &path, NULL, NULL) == SQLITE_OK) &&
891+
(sqlite3_create_function(db, "fpath", 0, SQLITE_UTF8, (void *) (uintptr_t) id, &fpath, NULL, NULL) == SQLITE_OK) &&
892+
(sqlite3_create_function(db, "epath", 0, SQLITE_UTF8, (void *) (uintptr_t) id, &epath, NULL, NULL) == SQLITE_OK) &&
893+
(sqlite3_create_function(db, "uidtouser", 1, SQLITE_UTF8, NULL, &uidtouser, NULL, NULL) == SQLITE_OK) &&
894+
(sqlite3_create_function(db, "gidtogroup", 1, SQLITE_UTF8, NULL, &gidtogroup, NULL, NULL) == SQLITE_OK) &&
895+
(sqlite3_create_function(db, "modetotxt", 1, SQLITE_UTF8, NULL, &modetotxt, NULL, NULL) == SQLITE_OK))?0:1;
871896
}
872897

873898
size_t print_results(sqlite3_stmt *res, FILE *out, const int printpath, const int printheader, const int printrows, const char *delim) {
@@ -936,7 +961,7 @@ size_t print_results(sqlite3_stmt *res, FILE *out, const int printpath, const in
936961
return rec_count;
937962
}
938963

939-
sqlite3 *open_aggregate(const char *name, const char *attach_name, const char *query) {
964+
sqlite3 *open_aggregate(const char *name, const char *attach_name, const char *query, int id) {
940965
// copy and sanitize query
941966
char buf[MAXSQL];
942967
SNPRINTF(buf, MAXSQL, "%s", query);
@@ -973,7 +998,7 @@ sqlite3 *open_aggregate(const char *name, const char *attach_name, const char *q
973998
GUFI_SQLITE_VFS) != SQLITE_OK) || // create the aggregate database
974999
(sqlite3_db_config(aggregate, SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, 1, NULL) != SQLITE_OK) || // enable extension loading
9751000
(sqlite3_extension_init(aggregate, &err_msg, NULL) != SQLITE_OK) || // load the sqlite3-pcre extension
976-
(addqueryfuncs(aggregate) != 0) || // this is needed to add some query functions like path() uidtouser() gidtogroup()
1001+
(addqueryfuncs(aggregate, id) != 0) || // this is needed to add some query functions like path() uidtouser() gidtogroup()
9771002
(sqlite3_exec(aggregate, esql, NULL, NULL, &err_msg) != SQLITE_OK) || // create the original entries table for the aggregate table to copy from
9781003
(sqlite3_exec(aggregate, create_results_table, NULL, NULL, &err_msg) != SQLITE_OK) || // create the aggregate table
9791004
(sqlite3_exec(aggregate, "DROP TABLE entries;", NULL, NULL, &err_msg) != SQLITE_OK) || // drop the entries table

‎src/gufi_dir2index.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,7 @@ struct work * validate_inputs() {
387387
}
388388

389389
void sub_help() {
390-
printf("input_dir walk this tree to produce GUFI-tree\n");
390+
printf("input_dir walk this tree to produce GUFI index\n");
391391
printf("output_dir build GUFI index here\n");
392392
printf("\n");
393393
}

‎src/gufi_query.c

+31-27
Original file line numberDiff line numberDiff line change
@@ -433,25 +433,11 @@ static size_t descend2(struct QPTPool *ctx,
433433
return pushed;
434434
}
435435

436-
/* ////////////////////////////////////////////////// */
437-
/* these functions need to be moved back into dbutils */
438-
static void path2(sqlite3_context *context, int argc, sqlite3_value **argv)
439-
{
440-
const size_t id = QPTPool_get_index((struct QPTPool *) sqlite3_user_data(context), pthread_self());
441-
sqlite3_result_text(context, gps[id].gpath, -1, SQLITE_TRANSIENT);
442-
443-
return;
444-
}
445-
446-
int addqueryfuncs2(sqlite3 *db, struct QPTPool * ctx) {
447-
return !(sqlite3_create_function(db, "path", 0, SQLITE_UTF8, ctx, &path2, NULL, NULL) == SQLITE_OK);
448-
}
449-
/* ////////////////////////////////////////////////// */
450-
451436
/* sqlite3_exec callback argument data */
452437
struct CallbackArgs {
453438
struct OutputBuffers * output_buffers;
454439
int id;
440+
size_t count;
455441
};
456442

457443
static int print_callback(void * args, int count, char **data, char **columns) {
@@ -508,6 +494,8 @@ static int print_callback(void * args, int count, char **data, char **columns) {
508494
}
509495

510496
free(lens);
497+
498+
ca->count++;
511499
/* } */
512500
return 0;
513501
}
@@ -679,7 +667,7 @@ int processdir(struct QPTPool * ctx, const size_t id, void * data, void * args)
679667
#endif
680668
/* this is needed to add some query functions like path() uidtouser() gidtogroup() */
681669
if (db) {
682-
addqueryfuncs2(db, ctx);
670+
addqueryfuncs(db, id);
683671
}
684672
#ifdef DEBUG
685673
clock_gettime(CLOCK_MONOTONIC, &addqueryfuncs_end);
@@ -747,8 +735,23 @@ int processdir(struct QPTPool * ctx, const size_t id, void * data, void * args)
747735
SNFORMAT_S(gps[id].gpath, MAXPATH, 1, shortname, strlen(shortname));
748736
/* printf("processdir: setting gpath = %s and gepath %s\n",gps[mytid].gpath,gps[mytid].gepath); */
749737
realpath(work->name,gps[id].gfpath);
750-
recs = rawquerydb(work->name, 1, db, in.sqlsum, 1, 0, 0, id);
751-
/* printf("summary ran %s on %s returned recs %d\n",in.sqlsum,work->name,recs); */
738+
739+
#ifndef NO_SQL_EXEC
740+
struct ThreadArgs * ta = (struct ThreadArgs *) args;
741+
struct CallbackArgs ca;
742+
ca.output_buffers = &ta->output_buffers;
743+
ca.id = id;
744+
ca.count = 0;
745+
746+
/* this probably needs a timer */
747+
char *err = NULL;
748+
if (sqlite3_exec(db, in.sqlsum, ta->print_callback_func, &ca, &err) != SQLITE_OK) {
749+
fprintf(stderr, "Error: %s: %s: \"%s\"\n", err, dbname, in.sqlent);
750+
sqlite3_free(err);
751+
}
752+
#endif
753+
754+
recs = ca.count;
752755
} else {
753756
recs = 1;
754757
}
@@ -963,7 +966,7 @@ int processdir(struct QPTPool * ctx, const size_t id, void * data, void * args)
963966
}
964967

965968
void sub_help() {
966-
printf("GUFI_tree find GUFI index-tree here\n");
969+
printf("GUFI_index find GUFI index here\n");
967970
printf("\n");
968971
}
969972

@@ -989,7 +992,7 @@ int main(int argc, char *argv[])
989992
/* but allow different fields to be filled at the command-line. */
990993
/* Callers provide the options-string for get_opt(), which will */
991994
/* control which options are parsed for each program. */
992-
int idx = parse_cmd_line(argc, argv, "hHT:S:E:an:o:d:O:I:F:y:z:G:J:e:m:B:w", 1, "GUFI_tree ...", &in);
995+
int idx = parse_cmd_line(argc, argv, "hHT:S:E:an:o:d:O:I:F:y:z:G:J:e:m:B:w", 1, "GUFI_index ...", &in);
993996
if (in.helped)
994997
sub_help();
995998
if (idx < 0)
@@ -1003,7 +1006,7 @@ int main(int argc, char *argv[])
10031006
struct ThreadArgs args;
10041007

10051008
/* initialize globals */
1006-
if (!outfiles_init(gts.outfd, in.outfile, in.outfilen, in.maxthreads + 1) ||
1009+
if (!outfiles_init(gts.outfd, in.outfile, in.outfilen, in.maxthreads) ||
10071010
!outdbs_init (gts.outdbd, in.outdb, in.outdbn, in.maxthreads, in.sqlinit, in.sqlinit_len) ||
10081011
!OutputBuffers_init(&args.output_buffers, in.maxthreads + 1, in.output_buffer_size)) {
10091012
return -1;
@@ -1040,22 +1043,23 @@ int main(int argc, char *argv[])
10401043
sqlite3 **intermediates = NULL;
10411044
char aggregate_name[MAXSQL] = {};
10421045
if (in.aggregate_or_print == AGGREGATE) {
1043-
/* modify in.sqlent to insert the results into the aggregate table */
1046+
/* modify in.sqlent to insert the results into the intermediate table (named 'aggregate') it is associated with */
10441047
char orig_sqlent[MAXSQL];
1045-
SNFORMAT_S(orig_sqlent, MAXSQL, 1, in.sqlent, strlen(in.sqlent));
1046-
SNFORMAT_S(in.sqlent, MAXSQL, 4, "INSERT INTO ", 12, AGGREGATE_ATTACH_NAME, strlen(AGGREGATE_ATTACH_NAME), ".entries ", (size_t) 9, orig_sqlent, strlen(orig_sqlent));
1048+
SNFORMAT_S(orig_sqlent, MAXSQL, 1, in.sqlent, in.sqlent_len);
1049+
SNFORMAT_S(in.sqlent, MAXSQL, 4, "INSERT INTO ", 12, AGGREGATE_ATTACH_NAME, strlen(AGGREGATE_ATTACH_NAME), ".entries ", (size_t) 9, orig_sqlent, in.sqlent_len);
10471050

10481051
/* create the aggregate database */
1052+
/* functions that use global values, like path might not work correctly */
10491053
SNPRINTF(aggregate_name, MAXSQL, AGGREGATE_NAME, -1);
1050-
if (!(aggregate = open_aggregate(aggregate_name, AGGREGATE_ATTACH_NAME, orig_sqlent))) {
1054+
if (!(aggregate = open_aggregate(aggregate_name, AGGREGATE_ATTACH_NAME, orig_sqlent, in.maxthreads))) {
10511055
return -1;
10521056
}
10531057

10541058
intermediates = (sqlite3 **) malloc(sizeof(sqlite3 *) * in.maxthreads);
10551059
for(int i = 0; i < in.maxthreads; i++) {
10561060
char intermediate_name[MAXSQL];
10571061
SNPRINTF(intermediate_name, MAXSQL, AGGREGATE_NAME, (int) i);
1058-
if (!(intermediates[i] = open_aggregate(intermediate_name, AGGREGATE_ATTACH_NAME, orig_sqlent))) {
1062+
if (!(intermediates[i] = open_aggregate(intermediate_name, AGGREGATE_ATTACH_NAME, orig_sqlent, i))) {
10591063
fprintf(stderr, "Could not open %s\n", intermediate_name);
10601064
cleanup_intermediates(intermediates, i);
10611065
closedb(aggregate);
@@ -1204,7 +1208,7 @@ int main(int argc, char *argv[])
12041208
#if defined(DEBUG) && defined(CUMULATIVE_TIMES) || BENCHMARK
12051209
const size_t rows =
12061210
#endif
1207-
OutputBuffers_flush_multiple(&args.output_buffers, in.maxthreads + 1, gts.outfd);
1211+
OutputBuffers_flush_multiple(&args.output_buffers, in.maxthreads, gts.outfd);
12081212

12091213
/* clean up globals */
12101214
OutputBuffers_destroy(&args.output_buffers, in.maxthreads + 1);

‎src/querydb.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ int main(int argc, char *argv[])
158158
);
159159

160160
//add query funcs to get uidtouser() gidtogroup() and path()
161-
addqueryfuncs(db);
161+
addqueryfuncs(db, 0);
162162

163163
// set the global path so path() is the path passed in
164164
memset(endname,0,sizeof(endname));

‎src/querydbn.c

+7-6
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,6 @@ void sub_help() {
100100
int main(int argc, char *argv[])
101101
{
102102
char name[MAXPATH];
103-
char dbname[MAXPATH];
104103
char rsqlstmt[MAXSQL];
105104
int rc;
106105
sqlite3 *db;
@@ -154,22 +153,24 @@ int main(int argc, char *argv[])
154153

155154

156155
//printf("processing query name %s numb dbs %d\n",name, numdbs);
157-
SNPRINTF(dbname,MAXPATH,"%s/%s",name,DBNAME);
158-
db = opendb(dbname, RDONLY, 1, 1,
156+
if (!(db = opendb(name, RDWR, 1, 1,
159157
NULL, NULL
160158
#ifdef DEBUG
161159
, NULL, NULL
162160
, NULL, NULL
163161
, NULL, NULL
164162
, NULL, NULL
165163
#endif
166-
);
164+
))) {
165+
fprintf(stderr, "Error: Unable to create database file \"%s\".\n", name);
166+
return -1;
167+
}
167168

168169
// add query funcs to get path() uidtouser() gidtogroup()
169-
addqueryfuncs(db);
170+
addqueryfuncs(db, 0);
170171

171172
// just zero out the global path so path() for this query is useless
172-
bzero(gps[0].gpath,sizeof(gps[0].gpath));
173+
memset(gps[0].gpath, 0, sizeof(gps[0].gpath));
173174

174175
SNPRINTF(sqlu,MAXSQL,"create temp view v%s as ",tabnam);
175176
i=0;

0 commit comments

Comments
 (0)