Make POOL_FLAG_ADDFILEPROVIDESFILTERED behaviour more standard

Turning on POOL_FLAG_ADDFILEPROVIDESFILTERED made the lookup of
all file dependencies lazy, not only the ones required by some
dependency. This changes the way searching works and may also
slow down some use cases.

We now changed addfileprovides() to record the needed non-standard
file dependencies and only make those lazy in createwhatprovides().

src/fileprovides.c

@@ -26,11 +26,29 @@
 struct searchfiles {
   Id *ids;
   int nfiles;
+  Id *nonstd_ids;
+  int nonstd_nfiles;
   Map seen;
 };
 
 #define SEARCHFILES_BLOCK 127
 
+static int
+finalize_nonstd_ids_cmp(const void *pa, const void *pb, void *dp) 
+{
+  Id a = *(Id *)pa;
+  Id b = *(Id *)pb;
+  return a - b; 
+}
+
+static void
+finalize_nonstd_ids(Pool *pool)
+{
+  pool->nonstd_ids = solv_realloc2(pool->nonstd_ids, pool->nonstd_nids, sizeof(Id));
+  if (pool->nonstd_nids > 1)
+    solv_sort(pool->nonstd_ids, pool->nonstd_nids, sizeof(Id), finalize_nonstd_ids_cmp, 0);
+}
+
 static void
 pool_addfileprovides_dep(Pool *pool, Id *ida, struct searchfiles *sf, struct searchfiles *isf)
 {
@@ -91,7 +109,12 @@ pool_addfileprovides_dep(Pool *pool, Id *ida, struct searchfiles *sf, struct sea
       if (*s != '/')
 	continue;
       if (csf != isf && pool->addedfileprovides == 1 && !repodata_filelistfilter_matches(0, s))
-	continue;	/* skip non-standard locations csf == isf: installed case */
+	{
+	  /* skip non-standard locations. csf == isf: installed case */
+	  csf->nonstd_ids = solv_extend(csf->nonstd_ids, csf->nonstd_nfiles, 1, sizeof(Id), SEARCHFILES_BLOCK);
+	  csf->nonstd_ids[csf->nonstd_nfiles++] = dep;
+	  continue;	/* skip non-standard locations csf == isf: installed case */
+	}
       csf->ids = solv_extend(csf->ids, csf->nfiles, 1, sizeof(Id), SEARCHFILES_BLOCK);
       csf->ids[csf->nfiles++] = dep;
     }
@@ -566,7 +589,8 @@ pool_addfileprovides_queue(Pool *pool, Queue *idq, Queue *idqinst)
   memset(&isf, 0, sizeof(isf));
   map_init(&isf.seen, pool->ss.nstrings + pool->nrels);
   pool->addedfileprovides = pool->addfileprovidesfiltered ? 1 : 2;
-
+  pool->nonstd_ids = solv_free(pool->nonstd_ids);
+  pool->nonstd_nids = 0;
   if (idq)
     queue_empty(idq);
   if (idqinst)
@@ -610,6 +634,13 @@ pool_addfileprovides_queue(Pool *pool, Queue *idq, Queue *idqinst)
 	queue_insertn(idqinst, idqinst->count, sf.nfiles, sf.ids);
       solv_free(sf.ids);
     }
+  if (sf.nonstd_nfiles)
+    {
+      POOL_DEBUG(SOLV_DEBUG_STATS, "found %d non-standard file dependencies\n", sf.nonstd_nfiles);
+      pool->nonstd_ids = sf.nonstd_ids;
+      pool->nonstd_nids= sf.nonstd_nfiles;
+      finalize_nonstd_ids(pool);
+    }
   if (isf.nfiles)
     {
 #if 0

src/pool.c

@@ -125,6 +125,7 @@ pool_free(Pool *pool)
   solv_free(pool->languagecache);
   solv_free(pool->errstr);
   solv_free(pool->rootdir);
+  solv_free(pool->nonstd_ids);
   solv_free(pool);
 }
 
@@ -573,7 +574,24 @@ pool_createwhatprovides(Pool *pool)
     POOL_DEBUG(SOLV_DEBUG_STATS, "whatprovidesaux memory used: %d K id array, %d K data\n", pool->whatprovidesauxoff / (int)(1024/sizeof(Id)), pool->whatprovidesauxdataoff / (int)(1024/sizeof(Id)));
 
   queue_empty(&pool->lazywhatprovidesq);
-  if ((!pool->addedfileprovides && pool->disttype == DISTTYPE_RPM) || pool->addedfileprovides == 1)
+  if (pool->addedfileprovides == 1)
+    {
+      /* lazyly add file provides for nonstd */
+      for (i = 0; i < pool->nonstd_nids; i++)
+	{
+	  Id id = pool->nonstd_ids[i];
+	  const char *str = pool->ss.stringspace + pool->ss.strings[id];
+	  if (str[0] != '/')		/* just in case, all entries should start with '/' */
+	    continue;
+	  /* setup lazy adding, but remember old value */
+	  if (pool->whatprovides[id] > 1)
+	    queue_push2(&pool->lazywhatprovidesq, id, pool->whatprovides[id]);
+	  pool->whatprovides[id] = 0;
+	  if (pool->whatprovidesaux)
+	    pool->whatprovidesaux[id] = 0;	/* sorry */
+	}
+    }
+  else if (!pool->addedfileprovides && pool->disttype == DISTTYPE_RPM)
     {
       if (!pool->addedfileprovides)
 	POOL_DEBUG(SOLV_DEBUG_STATS, "WARNING: pool_addfileprovides was not called, this may result in slow operation\n");
@@ -592,8 +610,9 @@ pool_createwhatprovides(Pool *pool)
 	  if (pool->whatprovidesaux)
 	    pool->whatprovidesaux[i] = 0;	/* sorry */
 	}
-      POOL_DEBUG(SOLV_DEBUG_STATS, "lazywhatprovidesq size: %d entries\n", pool->lazywhatprovidesq.count / 2);
     }
+  if (pool->lazywhatprovidesq.count)
+    POOL_DEBUG(SOLV_DEBUG_STATS, "lazywhatprovidesq size: %d entries\n", pool->lazywhatprovidesq.count / 2);
 
   POOL_DEBUG(SOLV_DEBUG_STATS, "createwhatprovides took %d ms\n", solv_timems(now));
 }

src/pool.h

@@ -163,6 +163,8 @@ struct s_Pool {
   Offset whatprovidesauxoff;
   Id *whatprovidesauxdata;
   Offset whatprovidesauxdataoff;
+  Id *nonstd_ids;		/* needed file provides not matching the standard filelist filter */
+  int nonstd_nids;
 
   int whatprovideswithdisabled;
 #endif