make random acquire queues work less random

Queues feeding workers like rred are created in a random pattern to get
a few of them to run in parallel – but if we already have an idling queue
we don't need to assign it to a (potentially new) random queue as that
saves us the (agruably small) overhead of starting up a new queue,
avoids adding jobs to an already busy queue while others idle and as
a bonus reduces the size of debug logs a bit.

We also keep starting new queues now until we reach our limit before
we assign work at random to them, which should give us a more effective
utilisation overall compared to potentially adding work to busy queues
while we haven't reached our queue limit yet.

diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index 757fab05797312972bd0e57a16a86f96480a9b6e..c65aef329b3d3123de076b24bf94e8ffb8aabc13 100644 (file)
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -352,23 +352,37 @@ string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
       return U.Access;
 
    string AccessSchema = U.Access + ':';
       return U.Access;
 
    string AccessSchema = U.Access + ':';

-       string FullQueueName;
+   string FullQueueName;

 
    if (U.Host.empty())
    {
 
    if (U.Host.empty())
    {

-      long randomQueue = random();
+      long existing = 0;
+      // check how many queues exist already and reuse empty ones
+      for (Queue const *I = Queues; I != 0; I = I->Next)
+        if (I->Name.compare(0, AccessSchema.length(), AccessSchema) == 0)
+        {
+           if (I->Items == nullptr)
+              return I->Name;
+           ++existing;
+        }
+

 #ifdef _SC_NPROCESSORS_ONLN
       long cpuCount = sysconf(_SC_NPROCESSORS_ONLN) * 2;
 #else
 #ifdef _SC_NPROCESSORS_ONLN
       long cpuCount = sysconf(_SC_NPROCESSORS_ONLN) * 2;
 #else

-      long cpuCount = _config->FindI("Acquire::QueueHost::Limit",10);
+      long cpuCount = 10;

 #endif
 #endif

-      if (cpuCount > 0)
-         randomQueue %= cpuCount;
+      cpuCount = _config->FindI("Acquire::QueueHost::Limit", cpuCount);

 
 

-      strprintf(FullQueueName, "%s%ld", AccessSchema.c_str(), randomQueue);
-      if (Debug) {
-         clog << "Chose random queue " << FullQueueName << " for " << Uri << endl;
+      if (cpuCount <= 0 || existing < cpuCount)
+        strprintf(FullQueueName, "%s%ld", AccessSchema.c_str(), existing);
+      else
+      {
+        long const randomQueue = random() % cpuCount;
+        strprintf(FullQueueName, "%s%ld", AccessSchema.c_str(), randomQueue);

       }
       }

+
+      if (Debug)
+         clog << "Chose random queue " << FullQueueName << " for " << Uri << endl;

    } else
    {
       FullQueueName = AccessSchema + U.Host;
    } else
    {
       FullQueueName = AccessSchema + U.Host;

diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 4e2d1f182a8bdbfee4bad808f460de9ce48c2b62..f219b9193e8804a70152a8a34277e624f4392157 100755 (executable)
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -30,7 +30,7 @@ echo 'hacked' > aptarchive/hacked-i386
 compressfile aptarchive/hacked-i386
 
 wasmergeused() {
 compressfile aptarchive/hacked-i386
 
 wasmergeused() {

-       testsuccess apt update "$@"
+       testsuccess apt update "$@" -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1 -o Debug::pkgAcquire=1

 
        msgtest 'No intermediate patch files' 'still exist'
        local EDS="$(find rootdir/var/lib/apt/lists -name '*.ed' -o -name '*.ed.*')"
 
        msgtest 'No intermediate patch files' 'still exist'
        local EDS="$(find rootdir/var/lib/apt/lists -name '*.ed' -o -name '*.ed.*')"


@@ -61,6 +61,8 @@ wasmergeused() {
        else
                msgpass
        fi
        else
                msgpass
        fi

+
+       testequal '1' grep -c rred:601 rootdir/tmp/testsuccess.output

 }
 
 testrun() {
 }
 
 testrun() {


@@ -214,7 +216,7 @@ SHA256-Download:
        cp Packages-future aptarchive/Packages
        rm -f rootdir/var/lib/apt/lists/*_Contents-*
        webserverconfig 'aptwebserver::overwrite::.*Contents-.*::filename' '/hacked-i386.gz'
        cp Packages-future aptarchive/Packages
        rm -f rootdir/var/lib/apt/lists/*_Contents-*
        webserverconfig 'aptwebserver::overwrite::.*Contents-.*::filename' '/hacked-i386.gz'

-       testfailure apt update "$@" -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1
+       testfailure apt update "$@"

        webserverconfig 'aptwebserver::overwrite::.*Contents-.*::filename' '/Contents-i386.gz'
        cp rootdir/tmp/testfailure.output patchdownload.output
        testfailure grep 'rred:600' patchdownload.output
        webserverconfig 'aptwebserver::overwrite::.*Contents-.*::filename' '/Contents-i386.gz'
        cp rootdir/tmp/testfailure.output patchdownload.output
        testfailure grep 'rred:600' patchdownload.output