FOSSology  3.2.0rc1
Open Source License Compliance by Open Source Software
adj2nest.c
Go to the documentation of this file.
1 /***************************************************************
2  adj2nest: Convert adjacency list to nested sets.
3 
4  Copyright (C) 2007-2015 Hewlett-Packard Development Company, L.P.
5  Copyright (C) 2015 Siemens AG
6 
7  This program is free software; you can redistribute it and/or
8  modify it under the terms of the GNU General Public License
9  version 2 as published by the Free Software Foundation.
10 
11  This program is distributed in the hope that it will be useful,
12  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  GNU General Public License for more details.
15 
16  You should have received a copy of the GNU General Public License along
17  with this program; if not, write to the Free Software Foundation, Inc.,
18  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 
20  ***************************************************************/
21 
78 #include <stdlib.h>
79 #include <stdio.h>
80 #include <unistd.h>
81 #include <string.h>
82 #include <ctype.h>
83 #include <signal.h>
84 #include <libgen.h>
85 
86 #include "libfossology.h"
87 
88 #define MAXCMD 4096
89 #define myBUFSIZ 2048
90 char SQL[256];
91 
92 #ifdef COMMIT_HASH_S
93 char BuildVersion[]="adj2nest build version: " VERSION_S " r(" COMMIT_HASH_S ").\n";
94 #else
95 char BuildVersion[]="adj2nest build version: NULL.\n";
96 #endif
97 
98 PGconn *pgConn = NULL;
99 
105 {
107  long Child;
108  long Sibling;
109 };
110 typedef struct uploadtree uploadtree;
111 uploadtree *Tree=NULL;
113 long TreeSize=0;
114 long TreeSet=0;
115 long SetNum=0;
116 int isBigUpload=0;
117 /************************************************************/
118 /************************************************************/
119 /************************************************************/
120 
126 void WalkTree (long Index, long Depth)
127 {
128  long LeftSet;
129  PGresult* pgResult;
130 
131  if (agent_verbose)
132  {
133  int i;
134  for(i=0; i<Depth; i++) printf(" ");
135  LOG_VERBOSE("%ld\n",Tree[Index].UploadtreePk);
136  }
137 
138  LeftSet = SetNum;
139  SetNum++;
140 
141  if (Tree[Index].Child > -1)
142  {
143  WalkTree(Tree[Index].Child,Depth+1);
144  SetNum++;
145  }
146 
147  snprintf(SQL,sizeof(SQL),"UPDATE %s SET lft='%ld', rgt='%ld' WHERE uploadtree_pk='%ld'",
148  uploadtree_tablename,LeftSet,SetNum,Tree[Index].UploadtreePk);
149  pgResult = PQexec(pgConn, SQL);
150  fo_checkPQcommand(pgConn, pgResult, SQL, __FILE__, __LINE__);
151  PQclear(pgResult);
153 
154  if (Tree[Index].Sibling > -1)
155  {
156  SetNum++;
157  WalkTree(Tree[Index].Sibling,Depth+1);
158  }
159 
160 } /* WalkTree() */
161 
168 void SetParent (long Parent, long Child)
169 {
170  long P;
171  static long LastParentId=-1;
172  static long LastParentIndex=-1;
173 
174  /* Insert the child */
175  Tree[TreeSet].UploadtreePk = Child;
176  TreeSet++;
177 
178  if (Parent == 0) /* ignore null parent */
179  {
180  return;
181  }
182 
183  /* Find the index of the parent */
184  if (Parent == LastParentId)
185  {
186  P = LastParentIndex;
187  }
188  else
189  {
190  P=0;
191  while((P<TreeSet) && (Tree[P].UploadtreePk != Parent)) { P++; }
192  if (P < TreeSet)
193  {
194  LastParentId = Parent;
195  LastParentIndex = P;
196  }
197  }
198 
199  if (P >= TreeSet)
200  {
201  /* Parent not found, so create it (right after the child). */
202  Tree[TreeSet].UploadtreePk = Parent;
203  Tree[TreeSet].Child = TreeSet-1;
204  LastParentId = Parent;
205  LastParentIndex = TreeSet;
206  TreeSet++;
207  return;
208  }
209 
210  /* Parent found, so follow the chain and add the child to the
211  end of the chain. */
212  if (Tree[P].Child < 0)
213  {
214  Tree[P].Child = TreeSet-1;
215  }
216  else
217  {
218  /* Already have a child so follow that child's sibling chain */
219  P=Tree[P].Child;
220  while(Tree[P].Sibling > -1) P=Tree[P].Sibling; /* find end of the chain */
221  Tree[P].Sibling = TreeSet-1;
222  }
223 } /* SetParent() */
224 
231 void LoadAdj (long UploadPk)
232 {
233  long i;
234  long Parent,Child;
235  long RootRows, NonRootRows;
236  PGresult* pgNonRootResult;
237  PGresult* pgRootResult;
238  PGresult* pgResult;
239  char LastChar;
240 
242 
243  /* If the last character of the uploadtree_tablename is a digit, run analyze */
244  LastChar = uploadtree_tablename[strlen(uploadtree_tablename)-1];
245  if (LastChar >= '0' && LastChar <= '9')
246  {
247  isBigUpload=1;
248  snprintf(SQL,sizeof(SQL),"ANALYZE %s",uploadtree_tablename);
249  pgResult = PQexec(pgConn, SQL);
250  fo_checkPQcommand(pgConn, pgResult, SQL, __FILE__ ,__LINE__);
251  PQclear(pgResult);
252  }
253 
254  snprintf(SQL,sizeof(SQL),"SELECT uploadtree_pk,parent FROM %s WHERE upload_fk = %ld AND parent IS NOT NULL ORDER BY parent, ufile_mode&(1<<29) DESC, ufile_name",uploadtree_tablename,UploadPk);
255  pgNonRootResult = PQexec(pgConn, SQL);
256  fo_checkPQresult(pgConn, pgNonRootResult, SQL, __FILE__, __LINE__);
257 
258  NonRootRows = PQntuples(pgNonRootResult);
259  TreeSize = NonRootRows;
260  LOG_VERBOSE("# Upload %ld: %ld items\n",UploadPk,TreeSize);
261 
262  snprintf(SQL,sizeof(SQL),"SELECT uploadtree_pk,parent FROM %s WHERE upload_fk = %ld AND parent IS NULL",uploadtree_tablename,UploadPk);
263  pgRootResult = PQexec(pgConn, SQL);
264  fo_checkPQresult(pgConn, pgRootResult, SQL, __FILE__, __LINE__);
265 
266  RootRows = PQntuples(pgRootResult);
267  TreeSize += RootRows;
268 
269  /* Got data! Populate the tree! */
270  if (Tree) { free(Tree); }
271  if (TreeSize <= 0) { Tree=NULL; return; }
272  Tree = (uploadtree *)calloc(TreeSize+1,sizeof(uploadtree));
273  for(i=0; i<TreeSize+1; i++)
274  {
275  Tree[i].UploadtreePk=-1;
276  Tree[i].Child=-1;
277  Tree[i].Sibling=-1;
278  }
279 
280  TreeSet=0;
281  SetNum=1;
282 
283  /* Load the roots */
284  for(i=0; i<RootRows; i++)
285  {
286  Child = atol(PQgetvalue(pgRootResult, i, 0));
287  Tree[TreeSet].UploadtreePk = Child;
288  TreeSet++;
289 
290  /* dummy heart to make sure the scheduler knows we are still alive */
291  if ((i % 100000) == 0) fo_scheduler_heart(0);
292  }
293 
294  /* Load all non-roots */
295  for(i=0; i<NonRootRows; i++)
296  {
297  Child = atol(PQgetvalue(pgNonRootResult,i,0));
298  Parent = atol(PQgetvalue(pgNonRootResult,i,1));
299  SetParent(Parent,Child);
300 
301  /* dummy heart to make sure the scheduler knows we are still alive */
302  if ((i % 100000) == 0) fo_scheduler_heart(0);
303  }
304 
305  /* Free up DB memory */
306  PQclear(pgNonRootResult);
307  PQclear(pgRootResult);
308  return;
309 } /* LoadAdj() */
310 
311 /*********************************************
312  Run on all uploads WHERE the upload
313  has no nested set numbers.
314  This displays each upload as it runs!
315  *********************************************/
316 void RunAllNew ()
317 {
318  int Row,MaxRow;
319  long UploadPk;
320  PGresult *pgResult;
321 
322  snprintf(SQL,sizeof(SQL), "SELECT DISTINCT upload_pk,upload_desc,upload_filename FROM upload WHERE upload_pk IN ( SELECT DISTINCT upload_fk FROM uploadtree WHERE lft IS NULL )");
323  pgResult = PQexec(pgConn, SQL);
324  fo_checkPQresult(pgConn, pgResult, SQL, __FILE__, __LINE__);
325 
326  MaxRow = PQntuples(pgResult);
327  for(Row=0; Row < MaxRow; Row++)
328  {
329  UploadPk = atol(PQgetvalue(pgResult,Row,0));
330  if (UploadPk >= 0)
331  {
332  char *S;
333  printf("Processing %ld :: %s",UploadPk,PQgetvalue(pgResult,Row,2));
334  S = PQgetvalue(pgResult,Row,1);
335  if (S && S[0]) printf(" (%s)",S);
336  printf("\n");
337  LoadAdj(UploadPk);
338  if (Tree) WalkTree(0,0);
339  if (Tree) free(Tree);
340  Tree=NULL;
341  TreeSize=0;
342  }
343  }
344  PQclear(pgResult);
345 } /* RunAllNew() */
346 
347 /*********************************************
348  ListUploads(): List every upload ID.
349  *********************************************/
350 void ListUploads ()
351 {
352  int Row,MaxRow;
353  long NewPid;
354  PGresult *pgResult;
355 
356  printf("# Uploads\n");
357  snprintf(SQL,sizeof(SQL), "SELECT upload_pk,upload_desc,upload_filename FROM upload ORDER BY upload_pk");
358  pgResult = PQexec(pgConn, SQL);
359  fo_checkPQresult(pgConn, pgResult, SQL, __FILE__, __LINE__);
360 
361  /* list each value */
362  MaxRow = PQntuples(pgResult);
363  for(Row=0; Row < MaxRow; Row++)
364  {
365  NewPid = atol(PQgetvalue(pgResult,Row,0));
366  if (NewPid >= 0)
367  {
368  char *S;
369  printf("%ld :: %s",NewPid,PQgetvalue(pgResult,Row,2));
370  S = PQgetvalue(pgResult,Row,1);
371  if (S && S[0]) printf(" (%s)",S);
372  printf("\n");
373  }
374  }
375  PQclear(pgResult);
376 } /* ListUploads() */
377 
378 
379 /************************************************************/
380 /************************************************************/
381 /************************************************************/
382 
383 /**********************************************
384  Given a string that contains
385  field='value' pairs, check if the field name
386  matches.
387  \param Field Haystack
388  \param S Needle
389  \return 1 on match, 0 on miss, -1 on no data.
390  **********************************************/
391 int MatchField (char *Field, char *S)
392 {
393  int Len;
394  if (!S || (S[0]=='\0')) return(-1);
395  while(isspace(S[0])) S++;
396  Len = strlen(Field);
397  if (!strncmp(Field,S,Len))
398  {
399  /* Matched string, now make sure it is a real match */
400  while (isspace(S[Len])) Len++;
401  if (S[Len] == '=') return (1);
402  }
403  return(0);
404 } /* MatchField() */
405 
406 /**********************************************
407  Given a string that contains
408  field='value' pairs, skip the first pair and
409  return the pointer to the next pair (or NULL if
410  end of string).
411  \param S field='value' pairs
412  \return Pointer to next pair
413  **********************************************/
414 char * SkipFieldValue (char *S)
415 {
416  char Quote;
417 
418  if (!S || (S[0]=='\0')) return(NULL);
419 
420  /* Skip the field */
421  while((S[0] != '\0') && (S[0]!='=')) S++; /* skip until the '=' is found */
422  if (S[0]=='\0') return(NULL);
423  S++; /* skip the '=' */
424  while(isspace(S[0])) S++; /* Skip any spaces */
425  if (S[0]=='\0') return(NULL);
426 
427  /* Now for the fun part... Skip the Value. This may be quoted. */
428  switch(S[0])
429  {
430  case '\"': case '\'':
431  Quote=S[0];
432  S++;
433  break;
434  default:
435  Quote=' ';
436  break;
437  }
438  while((S[0]!='\0') && (S[0]!=Quote))
439  {
440  if (S[0] == '\\') {
441  S += 2;
442  }
443  else S++;
444  }
445  if (S[0]==Quote) S++;
446  while(isspace(S[0])) S++; /* Skip any spaces */
447  return(S);
448 } /* SkipFieldValue() */
449 
450 /**********************************************
451  The scheduler taints field=value
452  pairs. Given a pair, return the untainted value.
453  NOTE: In string and out string CAN be the same string!
454  NOTE: strlen(Sout) is ALWAYS < strlen(Sin).
455  \param[in] Sin Tainted string
456  \param[out] Sout Untainted string
457  \return Untainted string or NULL if there is an error.
458  **********************************************/
459 char * UntaintValue (char *Sin, char *Sout)
460 {
461  char Quote;
462 
463  /* Skip the field */
464  while((Sin[0] != '\0') && (Sin[0]!='=')) Sin++; /* skip until the '=' is found */
465  if (Sin[0]=='\0') return(NULL);
466  Sin++; /* skip the '=' */
467  while(isspace(Sin[0])) Sin++; /* Skip any spaces */
468  if (Sin[0]=='\0') { Sout[0]='\0'; return(NULL); }
469 
470  /* The value may be inside quotes */
471  switch(Sin[0])
472  {
473  case '\"': case '\'':
474  Quote=Sin[0];
475  Sin++;
476  break;
477  default:
478  Quote=' ';
479  break;
480  }
481 
482  /* Now we're ready to untaint the value */
483  while((Sin[0]!='\0') && (Sin[0]!=Quote))
484  {
485  if (Sin[0] == '\\') {
486  Sin++; /* skip quote char */
487  if (Sin[0] == 'n') {
488  Sout[0] = '\n';
489  }
490  else if (Sin[0] == 'r') {
491  Sout[0] = '\r';
492  }
493  else if (Sin[0] == 'a') {
494  Sout[0] = '\a';
495  }
496  else {
497  Sout[0] = Sin[0];
498  }
499  Sout++;
500  Sin++; /* skip processed char */
501  }
502  else {
503  Sout[0] = Sin[0];
504  Sin++;
505  Sout++;
506  };
507  }
508  Sout[0]='\0'; /* terminate string */
509  return(Sout);
510 } /* UntaintValue() */
511 
512 /**********************************************
513  Convert field=value pairs into parameter.
514  This overwrites the parameter string!
515  The parameter is untainted from the scheduler.
516  \param ParmName field='value' pairs.
517  \param Parm Parameter to be set.
518  \returns 1 if Parm is set, 0 if not.
519  **********************************************/
520 int SetParm (char *ParmName, char *Parm)
521 {
522  int rc;
523  char *OldParm;
524  OldParm=Parm;
525  if (!ParmName || (ParmName[0]=='\0')) return(1); /* no change */
526  if (!Parm || (Parm[0]=='\0')) return(1); /* no change */
527 
528  /* Find the parameter */
529  while(!(rc=MatchField(ParmName,Parm)))
530  {
531  Parm = SkipFieldValue(Parm);
532  }
533  if (rc != 1) return(0); /* no match */
534 
535  /* Found it! Set the value */
536  UntaintValue(Parm,OldParm);
537  return(1);
538 } /* SetParm() */
539 
540 
548 int UpdateUpload(long UploadPk)
549 {
550  PGresult *pgResult;
551 
552  /* update upload.upload_mode to say that adj2nest was successful */
553  snprintf(SQL, sizeof(SQL), "UPDATE upload SET upload_mode = upload_mode | (1<<6) WHERE upload_pk='%ld'",
554  UploadPk);
555  pgResult = PQexec(pgConn, SQL); /* UPDATE upload */
556  if (fo_checkPQcommand(pgConn, pgResult, SQL, __FILE__ ,__LINE__)) return -1;
557  PQclear(pgResult);
558 
559  if(isBigUpload)
560  {
561  snprintf(SQL,sizeof(SQL),"VACUUM ANALYZE %s",uploadtree_tablename);
562  pgResult = PQexec(pgConn, SQL);
563  if (fo_checkPQcommand(pgConn, pgResult, SQL, __FILE__ ,__LINE__)) return -1;
564  PQclear(pgResult);
565  }
566  return(0);
567 }
568 
569 /*********************************************************
570  Usage of the agent
571  \param Name absolute path of the agent
572  *********************************************************/
573 void Usage (char *Name)
574 {
575  printf("Usage: %s [options] [id [id ...]]\n",Name);
576  printf(" -h :: help (print this message), then exit.\n");
577  printf(" -i :: initialize the database, then exit.\n");
578  printf(" -a :: run on ALL uploads that have no nested set records.\n");
579  printf(" -c SYSCONFDIR :: Specify the directory for the system configuration.\n");
580  printf(" -v :: verbose (-vv = more verbose).\n");
581  printf(" -u :: list all upload ids, then exit.\n");
582  printf(" no file :: process upload ids from the scheduler.\n");
583  printf(" id :: process upload ids from the command-line.\n");
584  printf(" -V :: print the version info, then exit.\n");
585 } /* Usage() */
586 
587 /*********************************************************/
588 int main (int argc, char *argv[])
589 {
590  int c, i, rv;
591  long UploadPk=-1;
592  long *uploads_to_scan;
593  int upload_count = 0;
594  int user_pk;
595  char *agent_desc = "Adj2nest Agent";
596  char *COMMIT_HASH;
597  char *VERSION;
598  char agent_rev[myBUFSIZ];
599 
600  /* connect to scheduler. Noop if not run from scheduler. */
601  fo_scheduler_connect(&argc, argv, &pgConn);
602 
603  COMMIT_HASH = fo_sysconfig("adj2nest", "COMMIT_HASH");
604  VERSION = fo_sysconfig("adj2nest", "VERSION");
605  sprintf(agent_rev, "%s.%s", VERSION, COMMIT_HASH);
606  /* Get the Agent Key from the DB */
607  fo_GetAgentKey(pgConn, basename(argv[0]), 0, agent_rev, agent_desc);
608 
609  /* for list of upload_pk's from the command line */
610  uploads_to_scan = calloc(argc, sizeof(long));
611 
612  /* Process command-line */
613  while((c = getopt(argc,argv,"aciuvVh")) != -1)
614  {
615  switch(c)
616  {
617  case 'a': /* run on ALL */
618  RunAllNew();
619  break;
620  case 'c':
621  break; /* handled by fo_scheduler_connect() */
622  case 'i':
623  PQfinish(pgConn);
624  return(0);
625  case 'v': agent_verbose++; break;
626  case 'u':
627  /* list ids */
628  ListUploads();
629  PQfinish(pgConn);
630  return(0);
631  case 'V':
632  printf("%s", BuildVersion);
633  PQfinish(pgConn);
634  return(0);
635  default:
636  Usage(argv[0]);
637  fflush(stdout);
638  PQfinish(pgConn);
639  exit(-1);
640  }
641  }
642 
643  /* Copy filename args (if any) into array */
644  for (i = optind; i < argc; i++)
645  {
646  uploads_to_scan[upload_count] = atol(argv[i]);
647  upload_count++;
648  }
649 
650  if (upload_count == 0)
651  {
652  user_pk = fo_scheduler_userID(); /* get user_pk for user who queued the agent */
653  while(fo_scheduler_next())
654  {
655  UploadPk = atol(fo_scheduler_current());
656 
657  /* Check Permissions */
658  if (GetUploadPerm(pgConn, UploadPk, user_pk) < PERM_WRITE)
659  {
660  LOG_ERROR("You have no update permissions on upload %ld", UploadPk);
661  continue;
662  }
663 
664  LoadAdj(UploadPk);
665  if (Tree) WalkTree(0,0);
666  if (Tree) free(Tree);
667  Tree=NULL;
668  TreeSize=0;
669  /* Update Upload */
670  rv = UpdateUpload(UploadPk);
671  if (rv == -1) LOG_ERROR("Unable to update mode on upload %ld", UploadPk);
672  } /* while() */
673  }
674  else
675  {
676  for (i = 0; i < upload_count; i++)
677  {
678  UploadPk = uploads_to_scan[i];
679  LoadAdj(UploadPk);
680  if (Tree) WalkTree(0,0);
681  if (Tree) free(Tree);
682  Tree=NULL;
683  TreeSize=0;
684  /* Update Upload */
685  rv = UpdateUpload(UploadPk);
686  if (rv == -1) LOG_ERROR("Unable to update mode on upload %ld", UploadPk);
687  }
688  free(uploads_to_scan);
689  }
690 
691  PQfinish(pgConn);
693  return 0;
694 } /* main() */
695 
FUNCTION char * GetUploadtreeTableName(PGconn *pgConn, int upload_pk)
Get the uploadtree table name for this upload_pk If upload_pk does not exist, return "uploadtree"...
Definition: libfossagent.c:421
int fo_checkPQresult(PGconn *pgConn, PGresult *result, char *sql, char *FileID, int LineNumb)
Check the result status of a postgres SELECT.
Definition: libfossdb.c:181
char BuildVersion[]
Definition: buckets.c:79
long UploadtreePk
Definition: adj2nest.c:106
long SetNum
Definition: adj2nest.c:115
PGconn * pgConn
Database connection.
Definition: adj2nest.c:98
char * fo_scheduler_current()
Get the last read string from the scheduler.
int UpdateUpload(long UploadPk)
Finish updating the upload record and permissions data.
Definition: adj2nest.c:548
void fo_scheduler_disconnect(int retcode)
Disconnect the scheduler connection.
void fo_scheduler_connect(int *argc, char **argv, PGconn **db_conn)
Establish a connection between an agent and the scheduler.
void SetParent(long Parent, long Child)
Definition: adj2nest.c:168
long Child
Definition: adj2nest.c:107
int agent_verbose
Common verbose flags for the agents, this is used so that the scheduler can change the verbose level ...
void WalkTree(long Index, long Depth)
Definition: adj2nest.c:126
void LoadAdj(long UploadPk)
Definition: adj2nest.c:231
int fo_checkPQcommand(PGconn *pgConn, PGresult *result, char *sql, char *FileID, int LineNumb)
Check the result status of a postgres commands (not select) If an error occured, write the error to s...
Definition: libfossdb.c:215
long TreeSet
Definition: adj2nest.c:114
char SQL[256]
SQL query to execute.
Definition: adj2nest.c:90
int fo_scheduler_userID()
Gets the id of the user that created the job that the agent is running.
Usage()
Print Usage statement.
Definition: fo_dbcheck.php:75
Contains information required by uploadtree elements.
Definition: adj2nest.c:104
The main FOSSology C library.
char * uploadtree_tablename
upload.uploadtree_tablename
Definition: adj2nest.c:112
FUNCTION int fo_GetAgentKey(PGconn *pgConn, const char *agent_name, long Upload_pk, const char *rev, const char *agent_desc)
Get the latest enabled agent key (agent_pk) from the database.
Definition: libfossagent.c:172
char * fo_scheduler_next()
Get the next data to process from the scheduler.
void fo_scheduler_heart(int i)
This function must be called by agents to let the scheduler know they are alive and how many items th...
FUNCTION int GetUploadPerm(PGconn *pgConn, long UploadPk, int user_pk)
Get users permission to this upload.
Definition: libfossagent.c:385
long Sibling
Definition: adj2nest.c:108
char * fo_sysconfig(const char *sectionname, const char *variablename)
gets a system configuration variable from the configuration data.
#define PERM_WRITE
Read-Write permission.
Definition: libfossology.h:45