nomos_utils.c

Go to the documentation of this file.

/***************************************************************
 Copyright (C) 2006-2014 Hewlett-Packard Development Company, L.P.

 This program is free software; you can redistribute it and/or
 modify it under the terms of the GNU General Public License
 version 2 as published by the Free Software Foundation.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License along
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

 ***************************************************************/

#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif /* not defined _GNU_SOURCE */

#include "nomos_utils.h"
#include "nomos.h"

#define FUNCTION

FUNCTION long add2license_ref(char *licenseName)
{

  PGresult *result;
  char query[myBUFSIZ];
  char insert[myBUFSIZ];
  char escLicName[myBUFSIZ];
  char *specialLicenseText;
  long rf_pk;

  int len;
  int error;
  int numRows;

  // escape the name
  len = strlen(licenseName);
  PQescapeStringConn(gl.pgConn, escLicName, licenseName, len, &error);
  if (error)
  LOG_WARNING("Does license name %s have multibyte encoding?", licenseName)

  /* verify the license is not already in the table */
  sprintf(query, "SELECT rf_pk FROM " LICENSE_REF_TABLE " where rf_shortname='%s'", escLicName);
  result = PQexec(gl.pgConn, query);
  if (fo_checkPQresult(gl.pgConn, result, query, __FILE__, __LINE__))
    return 0;
  numRows = PQntuples(result);
  if (numRows)
  {
    rf_pk = atol(PQgetvalue(result, 0, 0));
    PQclear(result);
    return rf_pk;
  }
  PQclear(result);

  /* Insert the new license */
  specialLicenseText = "License by Nomos.";

  sprintf(insert, "insert into license_ref(rf_shortname, rf_text, rf_detector_type) values('%s', '%s', 2)", escLicName,
      specialLicenseText);
  result = PQexec(gl.pgConn, insert);
  // ignore duplicate constraint failure (23505), report others
  if ((result == 0)
      || ((PQresultStatus(result) != PGRES_COMMAND_OK)
          && (strncmp(PG_ERRCODE_UNIQUE_VIOLATION, PQresultErrorField(result, PG_DIAG_SQLSTATE), 5))))
  {
    printf("ERROR: %s(%d): Nomos failed to add a new license. %s/n: %s/n",
    __FILE__, __LINE__, PQresultErrorMessage(result), insert);
    PQclear(result);
    return (0);
  }
  PQclear(result);

  /* retrieve the new rf_pk */
  result = PQexec(gl.pgConn, query);
  if (fo_checkPQresult(gl.pgConn, result, query, __FILE__, __LINE__))
    return 0;
  numRows = PQntuples(result);
  if (numRows)
    rf_pk = atol(PQgetvalue(result, 0, 0));
  else
  {
    printf("ERROR: %s:%s:%d Just inserted value is missing. On: %s", __FILE__, "add2license_ref()", __LINE__, query);
    PQclear(result);
    return (0);
  }
  PQclear(result);

  return (rf_pk);
}

FUNCTION long lrcache_hash(cacheroot_t *pcroot, char *rf_shortname)
{
  long hashval = 0;
  int len, i;

  /* use the first sizeof(long) bytes for the hash value */
  len = (strlen(rf_shortname) < sizeof(long)) ? strlen(rf_shortname) : sizeof(long);
  for (i = 0; i < len; i++)
    hashval += rf_shortname[i] << 8 * i;
  hashval = hashval % pcroot->maxnodes;
  return hashval;
}

FUNCTION void lrcache_print(cacheroot_t *pcroot)
{
  cachenode_t *pcnode;
  long hashval = 0;
  int i;

  pcnode = pcroot->nodes;
  for (i = 0; i < pcroot->maxnodes; i++)
  {
    if (pcnode->rf_pk != 0L)
    {
      hashval = lrcache_hash(pcroot, pcnode->rf_shortname);
      printf("%ld, %ld, %s\n", hashval, pcnode->rf_pk, pcnode->rf_shortname);
    }
    pcnode++;
  }
}

FUNCTION void lrcache_free(cacheroot_t *pcroot)
{
  cachenode_t *pcnode;
  int i;

  pcnode = pcroot->nodes;
  for (i = 0; i < pcroot->maxnodes; i++)
  {
    if (pcnode->rf_pk != 0L)
    {
      free(pcnode->rf_shortname);
    }
    pcnode++;
  }
  free(pcroot->nodes);
}

FUNCTION int lrcache_add(cacheroot_t *pcroot, long rf_pk, char *rf_shortname)
{
  cachenode_t *pcnode;
  long hashval = 0;
  int i;
  int noden;

  hashval = lrcache_hash(pcroot, rf_shortname);

  noden = hashval;
  for (i = 0; i < pcroot->maxnodes; i++)
  {
    noden = (hashval + i) & (pcroot->maxnodes - 1);

    pcnode = pcroot->nodes + noden;
    if (!pcnode->rf_pk)
    {
      pcnode->rf_shortname = strdup(rf_shortname);
      pcnode->rf_pk = rf_pk;
      break;
    }
  }
  if (i < pcroot->maxnodes)
    return 0;

  return -1; /* no space */
}

FUNCTION long lrcache_lookup(cacheroot_t *pcroot, char *rf_shortname)
{
  cachenode_t *pcnode;
  long hashval = 0;
  int i;
  int noden;

  hashval = lrcache_hash(pcroot, rf_shortname);

  noden = hashval;
  for (i = 0; i < pcroot->maxnodes; i++)
  {
    noden = (hashval + i) & (pcroot->maxnodes - 1);

    pcnode = pcroot->nodes + noden;
    if (!pcnode->rf_pk)
      return 0;
    if (strcmp(pcnode->rf_shortname, rf_shortname) == 0)
    {
      return pcnode->rf_pk;
    }
  }

  return 0; /* not found */
}

FUNCTION int initLicRefCache(cacheroot_t *pcroot)
{

  PGresult *result;
  char query[myBUFSIZ];
  int row;
  int numLics;

  if (!pcroot)
    return 0;

  sprintf(query, "SELECT rf_pk, rf_shortname FROM " LICENSE_REF_TABLE " where rf_detector_type=2");
  result = PQexec(gl.pgConn, query);
  if (fo_checkPQresult(gl.pgConn, result, query, __FILE__, __LINE__))
    return 0;

  numLics = PQntuples(result);
  /* populate the cache  */
  for (row = 0; row < numLics; row++)
  {
    lrcache_add(pcroot, atol(PQgetvalue(result, row, 0)), PQgetvalue(result, row, 1));
  }

  PQclear(result);

  return (1);
} /* initLicRefCache */

FUNCTION long get_rfpk(cacheroot_t *pcroot, char *rf_shortname)
{
  long rf_pk;
  size_t len;

  if ((len = strlen(rf_shortname)) == 0)
  {
    printf("ERROR! Nomos.c get_rfpk() passed empty name");
    return (0);
  }

  /* is this in the cache? */
  rf_pk = lrcache_lookup(pcroot, rf_shortname);
  if (rf_pk)
    return rf_pk;

  /* shortname was not found, so add it */
  /* add to the license_ref table */
  rf_pk = add2license_ref(rf_shortname);

  /* add to the cache */
  lrcache_add(pcroot, rf_pk, rf_shortname);

  return (rf_pk);
} /* get_rfpk */

FUNCTION char *getFieldValue(char *inStr, char *field, int fieldMax,
    char *value, int valueMax, char separator)
{
  int s;
  int f;
  int v;
  int gotQuote;

#ifdef PROC_TRACE
  traceFunc("== getFieldValue(inStr= %s fieldMax= %d separator= '%c'\n",
      inStr, fieldMax, separator);
#endif /* PROC_TRACE */

  memset(field, 0, fieldMax);
  memset(value, 0, valueMax);

  /* Skip initial spaces */
  while (isspace(inStr[0]))
  {
    inStr++;
  }

  if (inStr[0] == '\0')
  {
    return (NULL);
  }
  f = 0;
  v = 0;

  /* Skip to end of field name */
  for (s = 0; (inStr[s] != '\0') && !isspace(inStr[s]) && (inStr[s] != '='); s++)
  {
    field[f++] = inStr[s];
  }

  /* Skip spaces after field name */
  while (isspace(inStr[s]))
  {
    s++;
  }
  /* If it is not a field, then just return it. */
  if (inStr[s] != separator)
  {
    return (inStr + s);
  }
  if (inStr[s] == '\0')
  {
    return (NULL);
  }
  /* Skip '=' */
  s++;

  /* Skip spaces after '=' */
  while (isspace(inStr[s]))
  {
    s++;
  }
  if (inStr[s] == '\0')
  {
    return (NULL);
  }

  gotQuote = '\0';
  if ((inStr[s] == '\'') || (inStr[s] == '"'))
  {
    gotQuote = inStr[s];
    s++; /* skip quote */
    if (inStr[s] == '\0')
    {
      return (NULL);
    }
  }

  if (gotQuote)
  {
    for (; (inStr[s] != '\0') && (inStr[s] != gotQuote); s++)
    {
      if (inStr[s] == '\\')
      {
        value[v++] = inStr[++s];
      }
      else
      {
        value[v++] = inStr[s];
      }
    }
  }
  else
  {
    /* if it gets here, then there is no quote */
    for (; (inStr[s] != '\0') && !isspace(inStr[s]); s++)
    {
      if (inStr[s] == '\\')
      {
        value[v++] = inStr[++s];
      }
      else
      {
        value[v++] = inStr[s];
      }
    }
  }
  /* Skip spaces */
  while (isspace(inStr[s]))
  {
    s++;
  }

  return (inStr + s);
} /* getFieldValue */

FUNCTION void parseLicenseList()
{

  int numLics = 0;

  /* char saveLics[myBUFSIZ]; */
  char *saveptr = 0; /* used for strtok_r */
  char *saveLicsPtr;

  if ((strlen(cur.compLic)) == 0)
  {
    return;
  }

  /* check for a single name  FIX THIS!*/
  if (strstr(cur.compLic, ",") == NULL)
  {
    cur.licenseList[0] = cur.compLic;
    cur.licenseList[1] = NULL;
    return;
  }

  saveLicsPtr = strcpy(saveLics, cur.compLic);

  cur.tmpLics = strtok_r(saveLicsPtr, ",", &saveptr);

  cur.licenseList[numLics] = cur.tmpLics;
  numLics++;

  saveLicsPtr = NULL;
  while (cur.tmpLics)
  {
    cur.tmpLics = strtok_r(saveLicsPtr, ",", &saveptr);
    if (cur.tmpLics == NULL)
    {
      break;
    }
    cur.licenseList[numLics] = cur.tmpLics;
    numLics++;
  }
  cur.licenseList[numLics] = NULL;
  numLics++;

  /*
   int i;
   for(i=0; i<numLics; i++){
   printf("cur.licenseList[%d] is:%s\n",i,cur.licenseList[i]);
   }

   printf("parseLicenseList: returning\n");
   */

  return;
} /* parseLicenseList */

FUNCTION void Usage(char *Name)
{
  printf("Usage: %s [options] [file [file [...]]\n", Name);
  printf("  -h   :: help (print this message), then exit.\n");
  printf("  -i   :: initialize the database, then exit.\n");
  printf("  -c   :: specify the directory for the system configuration.\n");
  printf("  -l   :: print full file path (command line only).\n");
  printf("  -v   :: verbose (-vv = more verbose)\n");
  printf("  -J   :: output in JSON\n");
  printf("  -S   :: print Highlightinfo to stdout \n");
  printf("  file :: if files are listed, print the licenses detected within them.\n");
  printf("  no file :: process data from the scheduler.\n");
  printf("  -V   :: print the version info, then exit.\n");
  printf("  -d   :: specify a directory to scan.\n");
  printf("  -n   :: spaw n - 1 child processes to run, there will be n running processes(the parent and n - 1 children). \n the default n is 2(when n is less than 2 or not setting, will be changed to 2) when -d is specified.\n");
} /* Usage() */

FUNCTION void Bail(int exitval)
{
#ifdef PROC_TRACE
  traceFunc("== Bail(%d)\n", exitval);
#endif /* PROC_TRACE */

#if defined(MEMORY_TRACING) && defined(MEM_ACCT)
  if (exitval)
  {
    memCacheDump("Mem-cache @ Bail() time:");
  }
#endif /* MEMORY_TRACING && MEM_ACCT */

  /* close database and scheduler connections */
  if (gl.pgConn) {
    fo_dbManager_free(gl.dbManager);
    PQfinish(gl.pgConn);
  }
  fo_scheduler_disconnect(exitval);
  exit(exitval);
}

FUNCTION int optionIsSet(int val)
{
#ifdef PROC_TRACE
  traceFunc("== optionIsSet(%x)\n", val);
#endif /* PROC_TRACE */

  return (gl.progOpts & val);
} /* optionIsSet */

FUNCTION  void getFileLists(char *dirpath)
{
#ifdef PROC_TRACE
  traceFunc("== getFileLists(%s)\n", dirpath);
#endif /* PROC_TRACE */

  /*    listInit(&gl.sarchList, 0, "source-archives list & md5sum map"); */
  listInit(&cur.regfList, 0, "regular-files list");
  listInit(&cur.offList, 0, "buffer-offset list");
#ifdef FLAG_NO_COPYRIGHT
  listInit(&gl.nocpyrtList, 0, "no-copyright list");
#endif /* FLAG_NO_COPYRIGHT */

  listGetItem(&cur.regfList, cur.targetFile);
  return;
} /* getFileLists */

FUNCTION long updateLicenseFile(long rfPk)
{

  PGresult *result;

  if (rfPk <= 0)
  {
    return (-2);
  }

  /* If files are coming from command line instead of fossology repo,
   then there are no pfiles.  So don't update the db
   */
  if (cur.cliMode == 1)
    return (-1);

  result = fo_dbManager_ExecPrepared(
    fo_dbManager_PrepareStamement(
      gl.dbManager,
      "updateLicenseFile",
      "INSERT INTO license_file(rf_fk, agent_fk, pfile_fk) VALUES($1, $2, $3) RETURNING fl_pk",
      long, int, long
    ),
    rfPk, gl.agentPk, cur.pFileFk
  );

  if (result) {
    long licenseFileId = -1;
    if (PQntuples(result) > 0) {
      licenseFileId = atol(PQgetvalue(result, 0, 0));
    }

    PQclear(result);
    return (licenseFileId);
  } else {
    return (-1);
  }
} /* updateLicenseFile */

FUNCTION char convertIndexToHighlightType(int index)
{

  char type;

  if ((index >= _KW_first) && (index <= _KW_last))
    type = 'K';
  else if (index > _KW_last)
    type = 'L';
  else type = '0';

  return type;

}

FUNCTION int updateLicenseHighlighting(cacheroot_t *pcroot){

  /* If files are coming from command line instead of fossology repo,
   then there are no pfiles.  So don't update the db

   Also if we specifically do not want highlight information in the
   database skip this function
   */
  if(cur.cliMode == 1 || optionIsSet(OPTS_NO_HIGHLIGHTINFO ) ){
    return (TRUE);
  }
  PGresult *result;




#ifdef GLOBAL_DEBUG
  printf("%s %s %i \n", cur.filePath,cur.compLic , cur.theMatches->len);
#endif

  // This speeds up the writing to the database and ensures that we have either full highlight information or none
  PGresult* begin1 = PQexec(gl.pgConn, "BEGIN");
  PQclear(begin1);

  fo_dbManager_PreparedStatement* preparedKeywords;
  if(cur.keywordPositions->len > 0 ) {
    preparedKeywords = fo_dbManager_PrepareStamement(
      gl.dbManager,
      "updateLicenseHighlighting:keyword",
      "INSERT INTO highlight_keyword (pfile_fk, start, len) VALUES($1, $2, $3)",
      long, int, int
    );
  }
  int i;
  for (i = 0; i < cur.keywordPositions->len; ++i)
  {
    MatchPositionAndType* ourMatchv = getMatchfromHighlightInfo(cur.keywordPositions, i);
    result = fo_dbManager_ExecPrepared(
                preparedKeywords,
                cur.pFileFk, ourMatchv->start, ourMatchv->end - ourMatchv->start);
    if (result)
    {
      PQclear(result);
    }
  }
  PGresult* commit1 = PQexec(gl.pgConn, "COMMIT");
  PQclear(commit1);

  PGresult* begin2 =PQexec(gl.pgConn, "BEGIN");
  PQclear(begin2);
  fo_dbManager_PreparedStatement* preparedLicenses;

  if(cur.theMatches->len > 0 ) {
    preparedLicenses=fo_dbManager_PrepareStamement(
       gl.dbManager,
       "updateLicenseHighlighting",
       "INSERT INTO highlight (fl_fk, start, len, type) VALUES($1, $2, $3,'L')",
       long, int, int
     );
  }

  for (i = 0; i < cur.theMatches->len; ++i)
  {
    LicenceAndMatchPositions* ourLicence = getLicenceAndMatchPositions(cur.theMatches, i);

    int j;
    for (j = 0; j < ourLicence->matchPositions->len; ++j)
    {
      MatchPositionAndType* ourMatchv = getMatchfromHighlightInfo(ourLicence->matchPositions, j);
      if(ourLicence->licenseFileId == -1) {
        continue;
      }
      result = fo_dbManager_ExecPrepared(
                  preparedLicenses,
        ourLicence->licenseFileId,
        ourMatchv->start, ourMatchv->end - ourMatchv->start
      );
      if (result == NULL)
      {
        return (FALSE);
      } else {
        PQclear(result);
      }
    }
  }

  PGresult* commit2 = PQexec(gl.pgConn, "COMMIT");
  PQclear(commit2);
  return (TRUE);
} /* updateLicenseHighlighting */


FUNCTION void processFile(char *fileToScan)
{

  char *pathcopy;
#ifdef PROC_TRACE
  traceFunc("== processFile(%s)\n", fileToScan);
#endif /* PROC_TRACE */

  /* printf("   LOG: nomos scanning file %s.\n", fileToScan);  DEBUG */

  (void) strcpy(cur.cwd, gl.initwd);

  strcpy(cur.filePath, fileToScan);
  pathcopy = g_strdup(fileToScan);
  strcpy(cur.targetDir, dirname(pathcopy));
  g_free(pathcopy);
  strcpy(cur.targetFile, fileToScan);
  cur.targetLen = strlen(cur.targetDir);

  if (!isFILE(fileToScan))
  {
    LOG_FATAL("\"%s\" is not a plain file", fileToScan)
    Bail(-__LINE__);
  }

  getFileLists(cur.targetDir);
  listInit(&cur.fLicFoundMap, 0, "file-license-found map");
  listInit(&cur.parseList, 0, "license-components list");
  listInit(&cur.lList, 0, "license-list");

  processRawSource();

  /* freeAndClearScan(&cur); */
} /* Process File */

void setLicenseFileIdInHiglightArray(long licenseFileId, char* licenseName){
  int i;
  for (i = 0; i < cur.theMatches->len; ++i) {
    LicenceAndMatchPositions* ourLicence = getLicenceAndMatchPositions(cur.theMatches, i);
    if (strcmp(licenseName, ourLicence->licenceName) == 0)
      ourLicence->licenseFileId = licenseFileId;
  }
} /* setLicenseFileIdInHiglightArray */

int updateLicenseFileAndHighlightArray(char* licenseName, cacheroot_t* pcroot) {
  long rf_pk = get_rfpk(pcroot, licenseName);
  long licenseFileId = updateLicenseFile(rf_pk);
  if (licenseFileId > 0) {
    setLicenseFileIdInHiglightArray(licenseFileId, licenseName);
    return (true);
  } else {
    return (false);
  }
}

FUNCTION int recordScanToDB(cacheroot_t *pcroot, struct curScan *scanRecord)
{

  char *noneFound;
  int numLicenses;

#ifdef SIMULATESCHED
  /* BOBG: This allows a developer to simulate the scheduler
   with a load file for testing/debugging, without updating the
   database.  Like:
   cat myloadfile | ./nomos
   myloadfile is same as what scheduler sends:
   pfile_pk=311667 pfilename=9A96127E7D3B2812B50BF7732A2D0FF685EF6D6A.78073D1CA7B4171F8AFEA1497E4C6B33.183
   pfile_pk=311727 pfilename=B7F5EED9ECB679EE0F980599B7AA89DCF8FA86BD.72B00E1B419D2C83D1050C66FA371244.368
   etc.
   */
  printf("%s\n",scanRecord->compLic);
  return(0);
#endif

  noneFound = strstr(scanRecord->compLic, LS_NONE);
  if (noneFound != NULL)
  {
    if (!updateLicenseFileAndHighlightArray("No_license_found", pcroot))
      return (-1);
    return (0);
  }

  /* we have one or more license names, parse them */
  parseLicenseList();
  /* loop through the found license names */
  for (numLicenses = 0; cur.licenseList[numLicenses] != NULL; numLicenses++)
  {
    if (!updateLicenseFileAndHighlightArray(cur.licenseList[numLicenses], pcroot))
      return (-1);
  }

  if (updateLicenseHighlighting(pcroot) == FALSE)
  {
    printf("Failure in update of highlight table \n");
  }

  return (0);
} /* recordScanToDb */

FUNCTION inline MatchPositionAndType* getMatchfromHighlightInfo(GArray* in,
    int index)
{
  return &g_array_index(in, MatchPositionAndType, index);
}

FUNCTION inline LicenceAndMatchPositions* getLicenceAndMatchPositions(
    GArray* in, int index)
{
  return &g_array_index(in, LicenceAndMatchPositions, index);
}

FUNCTION void initializeCurScan(struct curScan* cur)
{
  cur->indexList =  g_array_new(FALSE, FALSE, sizeof(int));
  cur->theMatches = g_array_new(FALSE, FALSE, sizeof(LicenceAndMatchPositions));
  cur->keywordPositions = g_array_new(FALSE, FALSE, sizeof(MatchPositionAndType));
  cur->docBufferPositionsAndOffsets = g_array_new(FALSE, FALSE, sizeof(pairPosOff));
  cur->currentLicenceIndex=-1;
}


FUNCTION void freeAndClearScan(struct curScan *thisScan)
{
  /*
   Clear lists
   */
  listClear(&thisScan->regfList, DEALLOC_LIST);
  listClear(&thisScan->offList, DEALLOC_LIST);
  listClear(&thisScan->fLicFoundMap, DEALLOC_LIST);
  listClear(&thisScan->parseList, DEALLOC_LIST);
  listClear(&thisScan->lList, DEALLOC_LIST);
  g_array_free(thisScan->indexList,TRUE);
  cleanTheMatches(thisScan->theMatches);
  g_array_free(thisScan->keywordPositions, TRUE);
  g_array_free(thisScan->docBufferPositionsAndOffsets, TRUE);


  /* remove keys, data and hash table */
  hdestroy();

}

FUNCTION inline void cleanTheMatches(GArray* theMatches){

  int i;
  for(i=0; i< theMatches->len;  ++i) {
    cleanLicenceAndMatchPositions(getLicenceAndMatchPositions (theMatches , i));
  }
  g_array_free( theMatches, TRUE);
}

FUNCTION inline void cleanLicenceAndMatchPositions( LicenceAndMatchPositions* in )
{
  if(in->licenceName) g_free(in->licenceName);
  g_array_free(in->matchPositions, TRUE);
  g_array_free(in->indexList,TRUE);
}

FUNCTION inline void addLicence(GArray* theMatches, char* licenceName ) {
  LicenceAndMatchPositions newMatch;
  newMatch.indexList = cur.indexList;
  cur.indexList=g_array_new(FALSE, FALSE, sizeof(int));

  newMatch.matchPositions = g_array_new(FALSE, FALSE, sizeof(MatchPositionAndType));
  newMatch.licenceName = g_strdup(licenceName);
  newMatch.licenseFileId = -1; //initial Value <- check if it was set
  g_array_append_val(theMatches , newMatch);
}

inline void cleanLicenceBuffer(){
  g_array_set_size(cur.indexList, 0);
}

inline bool clearLastElementOfLicenceBuffer(){
  if(cur.indexList->len>0)
    g_array_remove_index(cur.indexList, cur.indexList->len -1);
  return true;
}