libfossrepo.c

Go to the documentation of this file.

/************************************************************
libfossrepo: A set of functions for accessing the file repository.

Copyright (C) 2007-2012 Hewlett-Packard Development Company, L.P.

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License version 2.1 as published by the Free Software Foundation.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public License
along with this library; if not, write to the Free Software Foundation, Inc.0
51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

************************************************************/

#include "libfossrepo.h"
#include "libfossscheduler.h"
#include "fossconfig.h"

#include <sys/stat.h>
#include <glib.h>

#ifndef FOSSREPO_CONF
#define FOSSREPO_CONF "/srv/fossology/repository"
#endif
#ifndef FOSSGROUP
#define FOSSGROUP "fossology"
#endif


#ifdef COMMIT_HASH
char LibraryRepoBuildVersion[]="Library libfossrepo Build version: " COMMIT_HASH ".\n";
#endif

#define MAXHOSTNAMELEN        64  
#define MAXLINE        1024       
#define REPONAME "REPOSITORY"     

#define GROUP 0                   

/*** Globals to simplify usage ***/
int RepDepth = 2;
char RepPath[MAXLINE + 1] = "";
#if GROUP
int RepGroup; 
#endif

#define REPCONFCHECK()        { if (!*RepPath) fo_RepOpen(); }


int _RepCheckType(const char* S)
{
  int i;
  if (S == NULL) return (0);
  for (i = 0; S[i] != '\0'; i++)
  {
    if (!isalnum(S[i]) && !strchr("@%_=+-", S[i])) return (0);
  }
  return (1);
} /* _RepCheckType() */

int _RepCheckString(char* S)
{
  int i;
  if (S == NULL) return (0);
  if (S[0] == '.') return (0);
  for (i = 0; S[i] != '\0'; i++)
  {
    if (!isalnum(S[i]) && !strchr("@%_.=+-", S[i])) return (0);
  }
  return (1);
} /* _RepCheckString() */

char* fo_RepGetRepPath()
{
  char* MyRepPath = NULL;

  REPCONFCHECK();

  /* allocate the path */
  MyRepPath = (char*) calloc(strlen(RepPath) + 1, 1);
  strcpy(MyRepPath, RepPath);
  return (MyRepPath);
} /* fo_RepGetRepPath() */

int fo_RepHostExist(char* Type, char* Host)
{
  char* entry;
  int i, length;
  GError* error;

  REPCONFCHECK();
  if (!_RepCheckType(Type)) return (-1);

  length = fo_config_list_length(sysconfig, "REPOSITORY", Host, &error);
  if (error)
  {
    fprintf(stderr, "ERROR: %s\n", error->message);
    return 0;
  }

  for (i = 0; i < length; i++)
  {
    entry = fo_config_get_list(sysconfig, "REPOSITORY", Host, i, &error);
    if (entry[0] == '*' || strncmp(Type, entry, strlen(Type)) == 0)
      return 1;
  }

  return 0;
} /* fo_RepHostExist() */

char* _RepGetHost(const char* Type, char* Filename, int MatchNum)
{
  char** hosts;
  char* entry;
  char* start;
  char* end;
  char* ret = NULL;
  int Match = 0;
  int i, j, kl, hl;
  GError* error = NULL;

  REPCONFCHECK();
  if (!_RepCheckType(Type) || !_RepCheckString(Filename))
    return (NULL);

  hosts = fo_config_key_set(sysconfig, REPONAME, &kl);
  for (i = 0; i < kl; i++)
  {
    hl = fo_config_list_length(sysconfig, REPONAME, hosts[i], &error);
    for (j = 0; j < hl; j++)
    {
      entry = fo_config_get_list(sysconfig, REPONAME, hosts[i], j, &error);
      char* remainder = NULL;
      strtok_r(entry, " ", &remainder);
      start = strtok_r(NULL, " ", &remainder);
      end = strtok_r(NULL, " ", &remainder);

      if (strcmp(entry, "*") == 0 || strcmp(entry, Type) == 0)
      {
        if ((strncasecmp(start, Filename, strlen(start)) <= 0) &&
          (strncasecmp(end, Filename, strlen(end)) >= 0))
        {
          Match++;
          if (Match == MatchNum)
          {
            ret = (char*) calloc(strlen(hosts[i]) + 1, sizeof(char));
            strcpy(ret, hosts[i]);
            g_free(entry);
            return ret;
          }
        }
      }

      g_free(entry);
    }
  }

  return NULL;
} /* _RepGetHost() */

char* fo_RepGetHost(char* Type, char* Filename)
{
  return (_RepGetHost(Type, Filename, 1));
} /* fo_RepGetHost() */

char* fo_RepMkPathTmp(const char* Type, char* Filename, char* Ext, int Which)
{
  char* Path;
  char* Host;
  int Len = 0;
  int i;
  int FilenameLen;

  if (!_RepCheckType(Type) || !_RepCheckString(Filename)) return (NULL);

  /* get the hostname */
  Host = _RepGetHost(Type, Filename, Which);
  if (Host)
  {Len += strlen(Host) + 1;}
  if (!Host && (Which > 1))
  {
    free(Host);
    return (NULL);
  }
  /* get the depth */
  if (Type) Len += strlen(Type) + 1;
  /* save the path too */
  Len += strlen(RepPath) + 1;

  /* add in depth */
  Len = Len + 3 * RepDepth;

  /* add in filename size */
  FilenameLen = strlen(Filename);
  Len += FilenameLen;
  if (Ext) Len += 1 + strlen(Ext);

  /* create the name */
  Path = (char*) calloc(Len + 1, 1);
  Len = 0; /* now Len is size of string */
  {
    strcat(Path, RepPath);
    strcat(Path, "/");
    Len += strlen(RepPath) + 1;
  }
  if (Host)
  {
    strcat(Path, Host);
    strcat(Path, "/");
    Len += strlen(Host) + 1;
  }
  if (Type)
  {
    strcat(Path, Type);
    strcat(Path, "/");
    Len += strlen(Type) + 1;
  }

  /* free memory */
  if (Host) free(Host);

  /* check if the filename is too small */
  if (FilenameLen < RepDepth * 2)
  {
    for (i = 0; i < FilenameLen; i++)
    {
      Path[Len++] = tolower(Filename[i]);
      if (i % 2 == 1) Path[Len++] = '/';
    }
    for (; i < RepDepth * 2; i++)
    {
      Path[Len++] = '_';
      if (i % 2 == 1) Path[Len++] = '/';
    }
  }
  else
  {
    /* add the filename */
    for (i = 0; i < RepDepth; i++)
    {
      Path[Len] = tolower(Filename[i * 2]);
      Path[Len + 1] = tolower(Filename[i * 2 + 1]);
      Path[Len + 2] = '/';
      Len += 3;
    }
  }

  for (i = 0; Filename[i] != '\0'; i++)
  {
    Path[Len] = tolower(Filename[i]);
    Len++;
  }

  if (Ext)
  {
    strcat(Path, ".");
    strcat(Path, Ext);
    Len += strlen(Type) + 1;
  }
  return (Path);
} /* fo_RepMkPathTmp() */

char* fo_RepMkPath(const char* Type, char* Filename)
{
  char* Path, * AltPath;
  int i;
  struct stat Stat;

  Path = fo_RepMkPathTmp(Type, Filename, NULL, 1);
  if (!Path) return (NULL);
  /* if something exists, then return it! */
  if (!stat(Path, &Stat))
  {return (Path);}

  /* Check if it exists in an alternate path */
  i = 2;
  while (1)
  {
    AltPath = fo_RepMkPathTmp(Type, Filename, NULL, i);
    if (!AltPath) return (Path); /* No alternate */
    /* If there is an alternate, return it. */
    if (!stat(AltPath, &Stat))
    {
      free(Path);
      return (AltPath);
    }
    i++;
  }

  /* should never get here */
  return (Path);
} /* fo_RepMkPath() */

void _RepUpdateTime(char* File)
{
  struct utimbuf Utime;
  Utime.actime = Utime.modtime = time(NULL);
  utime(File, &Utime);
} /* _RepUpdateTime() */

int _RepMkDirs(char* Fname)
{
  char Dir[FILENAME_MAX + 1];
  int i;
  int rc = 0;
  mode_t Mask;
#if GROUP
  gid_t Gid;
#endif

  memset(Dir, '\0', sizeof(Dir));
  strcpy(Dir, Fname);
  for (i = 1; Dir[i] != '\0'; i++)
  {
    if (Dir[i] == '/')
    {
      Dir[i] = '\0';
      Mask = umask(0000); /* mode: 0777 */
#if GROUP
      Gid = getegid();
      setegid(RepGroup);
#endif
      rc = mkdir(Dir, 0770); /* create this path segment */
#if GROUP
      setegid(Gid);
#endif
      umask(Mask);
      if (rc && (errno == EEXIST)) rc = 0;
      Dir[i] = '/';
      if (rc)
      {
        fprintf(stderr, "FATAL: 'mkdir %s' failed with rc=%d\n", Dir, rc);
        return (rc);
      }
    }
  }
  return (rc);
} /* _RepMkDirs() */

int fo_RepRenameTmp(char* Type, char* Filename, char* Ext)
{
  char* FnameOld, * Fname;
  int rc;

  FnameOld = fo_RepMkPathTmp(Type, Filename, Ext, 1);
  Fname = fo_RepMkPath(Type, Filename);
  if (!FnameOld || !Fname)
  {
    fprintf(stderr, "ERROR: Bad repository name: type='%s' name='%s'\n",
      Type, Filename);
    return (-1);
  }
  rc = rename(FnameOld, Fname);
  free(FnameOld);
  free(Fname);
  return (rc);
} /* fo_RepRenameTmp() */

int fo_RepExist(char* Type, char* Filename)
{
  char* Fname;
  struct stat Stat;
  int rc = 0;

  if (!_RepCheckType(Type))
  {
    fprintf(stderr, "ERROR: Invalid type '%s'\n", Type);
    return (-1);
  }
  if (!_RepCheckString(Filename))
  {
    fprintf(stderr, "ERROR: Invalid filename '%s'\n", Filename);
    return (-1);
  }

  Fname = fo_RepMkPath(Type, Filename);
  if (!Fname)
  {
    fprintf(stderr, "ERROR: Unable to allocate path for '%s/%s'\n", Type, Filename);
    return (-1);
  }
  if (!stat(Fname, &Stat)) rc = 1;
  free(Fname);
  return (rc);
} /* fo_RepExist() */

int fo_RepExist2(char* Type, char* Filename)
{
  char* Fname;
  struct stat Stat;
  int rc = 0;

  if (!_RepCheckType(Type))
  {
    fprintf(stderr, "ERROR: Invalid type '%s'\n", Type);
    return (-1);
  }
  if (!_RepCheckString(Filename))
  {
    fprintf(stderr, "ERROR: Invalid filename '%s'\n", Filename);
    return (-1);
  }

  Fname = fo_RepMkPath(Type, Filename);
  if (!Fname)
  {
    fprintf(stderr, "ERROR: Unable to allocate path for '%s/%s'\n", Type, Filename);
    return (-1);
  }
  if (stat(Fname, &Stat)) rc = errno;
  free(Fname);
  return (rc);
} /* fo_RepExist2() */

int fo_RepRemove(char* Type, char* Filename)
{
  char* Fname;
  struct stat Stat;
  int rc = 0;

  if (!_RepCheckType(Type))
  {
    fprintf(stderr, "ERROR: Invalid type '%s'\n", Type);
    return (0);
  }
  if (!_RepCheckString(Filename))
  {
    fprintf(stderr, "ERROR: Invalid filename '%s'\n", Filename);
    return (0);
  }

  Fname = fo_RepMkPath(Type, Filename);
  if (!Fname)
  {
    fprintf(stderr, "ERROR: Unable to allocate path for '%s/%s'\n", Type, Filename);
    return (0);
  }
  if (!stat(Fname, &Stat)) rc = unlink(Fname);
  free(Fname);
  return (rc);
} /* fo_RepRemove() */

int fo_RepFclose(FILE* F)
{
  if (!F) return (0);
  return (fclose(F));
} /* fo_RepFclose() */

FILE* fo_RepFread(char* Type, char* Filename)
{
  FILE* F = NULL;
  char* Fname;

  if (!_RepCheckType(Type))
  {
    fprintf(stderr, "ERROR: Invalid type '%s'\n", Type);
    return (NULL);
  }
  if (!_RepCheckString(Filename))
  {
    fprintf(stderr, "ERROR: Invalid filename '%s'\n", Filename);
    return (NULL);
  }

  Fname = fo_RepMkPath(Type, Filename);
  if (!Fname)
  {
    fprintf(stderr, "ERROR: Unable to allocate path for '%s/%s'\n", Type, Filename);
    return (NULL);
  }
  _RepUpdateTime(Fname);
  F = fopen(Fname, "rb");
  free(Fname);
  return (F);
} /* fo_RepFread() */

FILE* fo_RepFwriteTmp(char* Type, char* Filename, char* Ext)
{
  FILE* F = NULL;
  char* Fname;
  mode_t Mask;
#if GROUP
  gid_t Gid;
#endif

  if (!_RepCheckType(Type))
  {
    fprintf(stderr, "ERROR: Invalid type '%s'\n", Type);
    return (NULL);
  }
  if (!_RepCheckString(Filename))
  {
    fprintf(stderr, "ERROR: Invalid filename '%s'\n", Filename);
    return (NULL);
  }

  Fname = fo_RepMkPathTmp(Type, Filename, Ext, 1);
  if (!Fname)
  {
    fprintf(stderr, "ERROR: Unable to allocate path for '%s/%s'\n", Type, Filename);
    return (NULL);
  }
  if (_RepMkDirs(Fname))
  {
    free(Fname);
    return (NULL);
  }
  _RepUpdateTime(Fname);
  Mask = umask(0117); /* mode: 0660 */
#if GROUP
  Gid = getegid();
  setegid(RepGroup);
#endif
  F = fopen(Fname, "wb");
  if (!F)
  {
    fprintf(stderr, "ERROR: %s, in %s:%d, failed to open [%s]\n",
      strerror(errno), __FILE__, __LINE__, Fname);
    free(Fname);
    return (NULL);
  }
  chmod(Fname, S_ISGID | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); /* when umask fails */
#if GROUP
  setegid(Gid);
#endif
  umask(Mask);
  free(Fname);
  return (F);
} /* fo_RepFwriteTmp() */

FILE* fo_RepFwrite(char* Type, char* Filename)
{
  return (fo_RepFwriteTmp(Type, Filename, NULL));
} /* fo_RepFwrite() */

void fo_RepMunmap(RepMmapStruct* M)
{
  if (!M) return;
  if (M->_MmapSize > 0) munmap(M->Mmap, M->_MmapSize);
  close(M->FileHandle);
  free(M);
} /* fo_RepMunmap() */

RepMmapStruct* fo_RepMmapFile(char* Fname)
{
  RepMmapStruct* M;
  struct stat Stat;
  int PageSize;

  M = (RepMmapStruct*) calloc(1, sizeof(RepMmapStruct));
  if (!M)
  {return (NULL);}

  /* open the file (memory map) */
  M->FileHandle = open(Fname, O_RDONLY);
  if (M->FileHandle == -1)
  {
    fprintf(stderr, "ERROR: Unable to open file for mmap (%s)\n", Fname);
    free(M);
    return (NULL);
  }

  /* find how big the file is (to allocate it) */
  if (fstat(M->FileHandle, &Stat) == -1)
  {
    fprintf(stderr, "ERROR: Unable to stat file (%s)\n", Fname);
    close(M->FileHandle);
    free(M);
    return (NULL);
  }
  PageSize = getpagesize();

  /* only mmap the first 1G */
  if (Stat.st_size > 0x7fffffff) Stat.st_size = 0x80000000;

  M->MmapSize = Stat.st_size;
  M->_MmapSize = M->MmapSize + PageSize - (M->MmapSize % PageSize);
  M->Mmap = mmap(0, M->_MmapSize, PROT_READ, MAP_PRIVATE, M->FileHandle, 0);
  if (M->Mmap == MAP_FAILED)
  {
    fprintf(stderr, "ERROR: Unable to mmap file (%s)\n", Fname);
    close(M->FileHandle);
    free(M);
    return (NULL);
  }
  return (M);
} /* fo_RepMmapFile() */

RepMmapStruct* fo_RepMmap(char* Type, char* Filename)
{
  RepMmapStruct* M;
  char* Fname;

  if (!_RepCheckType(Type) || !_RepCheckString(Filename)) return (NULL);

  Fname = fo_RepMkPath(Type, Filename);
  if (!Fname) return (NULL);
  _RepUpdateTime(Fname);

  M = fo_RepMmapFile(Fname);
  free(Fname);
  return (M);
} /* fo_RepMmap() */

int fo_RepImport(char* Source, char* Type, char* Filename, int Link)
{
  if (0 == strcmp(Type, "files"))
  {
    chmod(Source, S_ISGID | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH); /* change mode */
  }
  /*** code uses read/write ***/
  /*** Could use mmap, but it isn't noticably faster and could have
  problems with multi-gig files ***/
  int LenIn, LenOut;
  int i;
  char Buf[0x80000]; /* 80K blocks */
  char vBuf[0x80000]; /* 80K blocks */
  FILE* Fin;
  FILE* Fout;
  char* FoutPath;

  /* easy route: make a hard link */
  if (Link)
  {
    FoutPath = fo_RepMkPath(Type, Filename);
    if (!FoutPath) return (0);
    if (_RepMkDirs(FoutPath)) /* make the directory */
    {
      free(FoutPath);
      return (1);
    }
    if (link(Source, FoutPath) == 0)
    {
      free(FoutPath);
      return (0);
    }
    free(FoutPath);
  } /* try a hard link */

  /* hard route: actually copy the file */
  Fin = fopen(Source, "rb");
  if (!Fin)
  {
    fprintf(stderr, "ERROR: Unable to open source file '%s'\n", Source);
    return (1);
  }
  setvbuf(Fin, vBuf, _IOFBF, sizeof(vBuf));

  Fout = fo_RepFwriteTmp(Type, Filename, "I"); /* tmp = ".I" for importing... */
  if (!Fout)
  {
    fprintf(stderr, "ERROR: Invalid -- type='%s' filename='%s'\n", Type, Filename);
    fclose(Fin);
    return (2);
  }

  LenIn = fread(Buf,1,sizeof(Buf),Fin);
  while(LenIn > 0)
  {
    LenOut=0;
    while(LenOut < LenIn)
    {
      i = fwrite(Buf+LenOut,1,LenIn - LenOut,Fout);
      LenOut += i;
      if (i == 0)
      {
        /*** Oh no!  Write failed! ***/
        fclose(Fout);
        fo_RepFclose(Fout);
        fo_RepRemove(Type,Filename);
        fprintf(stderr,"ERROR: Write failed -- type='%s' filename='%s'\n",Type,Filename);
        fclose(Fin);
        return(3);
      }
    }
    LenIn = fread(Buf,1,sizeof(Buf),Fin);
  }
  fo_RepFclose(Fout);
  fclose(Fin);
  fo_RepRenameTmp(Type, Filename, "I"); /* mv .I to real name */
  return (0);
} /* fo_RepImport() */

void fo_RepClose()
{
  RepDepth = 2; /* default depth */
  memset(RepPath, '\0', sizeof(RepPath));
  RepPath[0] = '.'; /* default to local directory */
} /* fo_RepClose() */

int fo_RepOpen()
{
  return fo_RepOpenFull(sysconfig);
}

int fo_RepOpenFull(fo_conf* config)
{
  GError* error = NULL;
  char* path;

#if GROUP
  struct group *Group;
  gid_t Gid;
#endif

  fo_RepClose(); /* reset everything */

#if GROUP
  /* Make sure we can use group */
  Group = getgrnam(FOSSGROUP);
  if (!Group) return(0);  /* no such group */
  RepGroup = Group->gr_gid;
  Gid = getegid();
  if ((Gid != RepGroup) && setegid(RepGroup))
  {
    perror("Huh?");
    return(0);
  }
  setegid(Gid);
#endif

  /* Load the depth configuration */
  char* repDepthStr = fo_config_get(config, "FOSSOLOGY", "depth", &error);
  if (error)
  {
    fprintf(stderr, "ERROR %s.%d: %s\n", __FILE__, __LINE__, error->message);
    return 0;
  }
  RepDepth = atoi(repDepthStr);

  /* Load the path configuration */
  path = fo_config_get(config, "FOSSOLOGY", "path", &error);
  if (error)
  {
    fprintf(stderr, "ERROR %s.%d: %s\n", __FILE__, __LINE__, error->message);
    return 0;
  }
  strncpy(RepPath, path, sizeof(RepPath)-1);
  RepPath[sizeof(RepPath)-1] = 0;

  return 1;
} /* fo_RepOpen() */

char* fo_RepValidate(fo_conf* config)
{
  char* retval = NULL;
  int32_t nhosts, nlist, i, j;
  char* gname = "REPOSITORY";
  char** hosts;
  char* curr;
  GRegex* regex = NULL;
  GMatchInfo* match = NULL;
  uint32_t begin, end;
  gchar* begin_str;
  gchar* end_str;


  if ((hosts = fo_config_key_set(config, gname, &nhosts)) == NULL)
    return g_strdup("The fossology.conf file does not contain a \"REPOSITORY\" group.");

  /* Regex to match repository lines in the configuration file.
   *
   * This will match a file type followed by two hexidecimal numbers. Possible
   * file types are gold, files, logs, license, test, and all type (denoted by
   * a *).
   *
   * example match:
   *   * 00 ff
   */
  regex = g_regex_new(
    "(\\*|gold|files|logs|license|test)\\s+([[:xdigit:]]+)\\s+([[:xdigit:]]+)$",
    0, 0, NULL);

  for (i = 0; i < nhosts; i++)
  {
    nlist = fo_config_list_length(config, gname, hosts[i], NULL);

    for (j = 0; j < nlist; j++)
    {
      curr = fo_config_get_list(config, gname, hosts[i], j, NULL);

      if (!g_regex_match(regex, curr, 0, &match))
      {
        retval = g_strdup_printf("%s[] = %s", hosts[i], curr);
        break;
      }

      begin_str = g_match_info_fetch(match, 2);
      end_str = g_match_info_fetch(match, 3);

      begin = strtoul(begin_str, NULL, 16);
      end = strtoul(end_str, NULL, 16);

      if (begin >= end)
      {
        retval = g_strdup_printf("%s[] = %s", hosts[i], curr);
        break;
      }

      g_free(begin_str);
      g_free(end_str);
      g_match_info_free(match);
    }
  }

  g_regex_unref(regex);
  return retval;
} /* fo_RepValidate() */