Skip to content
Snippets Groups Projects
  1. Feb 19, 2025
  2. Feb 18, 2025
  3. Feb 17, 2025
  4. Feb 11, 2025
  5. Feb 06, 2025
  6. Feb 04, 2025
  7. Feb 03, 2025
  8. Feb 04, 2025
  9. Feb 03, 2025
  10. Feb 18, 2025
  11. Feb 04, 2025
  12. Feb 17, 2025
  13. Jan 30, 2025
  14. Jan 29, 2025
    • Pierre-Yves David's avatar
      stream-clone-v2: directly use the os module for file operation · c124308e3cd4
      Pierre-Yves David authored
      Sadly this is still significantly faster. I am not sure what the python file
      object does internally, but bypassing it provide a good speedup. Probably
      because we spend less time with the GIL.
      
      This is the end of this first round of optimization. Overall, the threaded
      version is almost twice faster as the baseline on our benchmark machine (and
      three time faster than the naive implementation) (see first batch of benchmark
      below).
      
      In reality, various improvement we did also help the non-threaded case. Our
      benchmark show about 30% improvement after this series. (second benchmark batch
      below)
      
      However, the threaded version remains faster in our benchmark (third benchmark).
      
      Further improvements can probably be achieved by speeding up the directories
      creation process and but moving to a compiled language (both faster in general
      and without a GIL).
      
      ########### Comparing evolution of the threaded code ##########################
      
      ### benchmark.name                         = hg.perf.exchange.stream.consume
        # bin-env-vars.hg.flavor                 = default
        # bin-env-vars.hg.py-re2-module          = default
        # benchmark.variants.memory-target       = default
        # benchmark.variants.num-writer          = default
        # benchmark.variants.parallel-processing = yes
        # benchmark.variants.progress            = no
        # benchmark.variants.read-from-memory    = yes
        # benchmark.variants.version             = v2
       ## data-env-vars.name                     = mercurial-public-2024-03-22-zstd-sparse-revlog
      baseline:     0.249693  ~~~~~
      naive-thread: 0.305973  (+22.54%, +0.06)
      prev-change:  0.154647  (-38.07%, -0.10)
      this-change:  0.143486  (-42.54%, -0.11)
       ## data-env-vars.name                     = netbeans-2019-11-07-zstd-sparse-revlog
      baseline:     13.136674  ~~~~~
      naive-thread: 18.467590  (+40.58%, +5.33)
      prev-change:  7.843554  (-40.29%, -5.29)
      this-change:  7.423071  (-43.49%, -5.71)
       ## data-env-vars.name                     = netbsd-xsrc-all-2024-09-19-zstd-sparse-revlog
      baseline:     5.317709  ~~~~~
      naive-thread: 7.338505  (+38.00%, +2.02)
      prev-change:  3.040664  (-42.82%, -2.28)
      this-change:  2.776906  (-47.78%, -2.54)
       ## data-env-vars.name                     = netbsd-xsrc-draft-2024-09-19-zstd-sparse-revlog
      baseline:     5.398368  ~~~~~
      naive-thread: 7.333354  (+35.84%, +1.93)
      prev-change:  3.070976  (-43.11%, -2.33)
      this-change:  2.795593  (-48.21%, -2.60)
       ## data-env-vars.name                     = pypy-2024-03-22-zstd-sparse-revlog
      baseline:     3.acbb55  ~~~~~
      naive-thread: 4.238172  (+38.11%, +1.17)
      prev-change:  1.832118  (-40.30%, -1.24)
      this-change:  1.640016  (-46.56%, -1.43)
       ## data-env-vars.name                     = heptapod-public-2024-03-25-zstd-sparse-revlog
      baseline:     7.244015  ~~~~~
      naive-thread: 9.901032  (+36.68%, +2.66)
      prev-change:  4.478754  (-38.17%, -2.77)
      this-change:  3.946716  (-45.52%, -3.30)
       ## data-env-vars.name                     = mozilla-central-2024-03-22-zstd-sparse-revlog
      baseline:     51.934795  ~~~~~
      naive-thread: 78.194540 (+50.56%, +26.26)
      prev-change:  30.349379 (-41.56%, -21.59)
      this-change:  27.228980 (-47.57%, -24.71)
       ## data-env-vars.name                     = mozilla-unified-2024-03-22-zstd-sparse-revlog
      baseline:     52.253858  ~~~~~
      naive-thread: 77.492938 (+48.30%, +25.24)
      prev-change:  31.179906 (-40.33%, -21.07)
      this-change:  28.110867 (-46.20%, -24.14)
       ## data-env-vars.name                     = mozilla-try-2024-03-26-zstd-sparse-revlog
        # benchmark.variants.read-from-memory    = no
      baseline:     130.584329  ~~~~~
      naive-thread: 164.366925 (+25.87%, +33.78)
      prev-change:  91.454508 (-29.97%, -39.13)
      this-change:  91.002420 (-30.31%, -39.58)
      
      
      ########### Comparing evolution of the non-threaded code ######################
      
      ### benchmark.name                         = hg.perf.exchange.stream.consume
        # bin-env-vars.hg.flavor                 = default
        # bin-env-vars.hg.py-re2-module          = default
        # benchmark.variants.memory-target       = default
        # benchmark.variants.num-writer          = default
        # benchmark.variants.parallel-processing = no
        # benchmark.variants.progress            = no
        # benchmark.variants.read-from-memory    = yes
        # benchmark.variants.version             = v2
       ## data-env-vars.name                     = mercurial-public-2024-03-22-zstd-sparse-revlog
      base-line:   0.247757  ~~~~~
      this-change: 0.168695  (-31.91%, -0.08)
      ### data-env-vars.name                     = netbeans-2019-11-07-zstd-sparse-revlog
      base-line:   13.195582  ~~~~~
      this-change: 9.240878  (-29.97%, -3.95)
       ## data-env-vars.name                     = netbsd-xsrc-all-2024-09-19-zstd-sparse-revlog
      base-line:   5.369459  ~~~~~
      this-change: 3.637088  (-32.26%, -1.73)
       ## data-env-vars.name                     = netbsd-xsrc-draft-2024-09-19-zstd-sparse-revlog
      base-line:   5.345086  ~~~~~
      this-change: 3.587604  (-32.88%, -1.76)
       ## data-env-vars.name                     = pypy-2024-03-22-zstd-sparse-revlog
      base-line:   3.110852  ~~~~~
      this-change: 2.069172  (-33.49%, -1.04)
       ## data-env-vars.name                     = heptapod-public-2024-03-25-zstd-sparse-revlog
      base-line:   7.293048  ~~~~~
      this-change: 4.931636  (-32.38%, -2.36)
       ## data-env-vars.name                     = mozilla-central-2024-03-22-zstd-sparse-revlog
      base-line:   52.067856  ~~~~~
      this-change: 35.360483 (-32.09%, -16.71)
       ## data-env-vars.name                     = mozilla-unified-2024-03-22-zstd-sparse-revlog
      base-line:   52.347182  ~~~~~
      this-change: 35.465007 (-32.25%, -16.88)
       ## data-env-vars.name                     = mozilla-try-2024-03-26-zstd-sparse-revlog
        # benchmark.variants.read-from-memory    = no
      base-line:   143.083922  ~~~~~
      this-change: 107.609183 (-24.79%, -35.47)
      
      
      ########### Comparing non-threaded vs threaded ################################
      
      ### benchmark.name                      = hg.perf.exchange.stream.consume
        # bin-env-vars.hg.flavor              = default
        # bin-env-vars.hg.py-re2-module       = default
        # bin-env-vars.hg.changeset.node      = fec3492130e0
        # benchmark.variants.memory-target    = default
        # benchmark.variants.num-writer       = default
        # benchmark.variants.progress         = no
        # benchmark.variants.read-from-memory = yes
        # benchmark.variants.version          = v2
       ## data-env-vars.name                  = mercurial-public-2024-03-22-zstd-sparse-revlog
      not-threaded: 0.168695  ~~~~~
      threaded:     0.143486  (-14.94%, -0.03)
      ### data-env-vars.name                  = netbeans-2019-11-07-zstd-sparse-revlog
      not-threaded: 9.240878  ~~~~~
      threaded:     7.423071  (-19.67%, -1.82)
      ### data-env-vars.name                  = netbsd-xsrc-all-2024-09-19-zstd-sparse-revlog
      not-threaded: 3.637088  ~~~~~
      threaded:     2.776906  (-23.65%, -0.86)
      ### data-env-vars.name                  = netbsd-xsrc-draft-2024-09-19-zstd-sparse-revlog
      not-threaded: 3.587604  ~~~~~
      threaded:     2.795593  (-22.08%, -0.79)
      ### data-env-vars.name                  = pypy-2024-03-22-zstd-sparse-revlog
      not-threaded: 2.069172  ~~~~~
      threaded:     1.640016  (-20.74%, -0.43)
      ## data-env-vars.name                  = heptapod-public-2024-03-25-zstd-sparse-revlog
      not-threaded: 4.931636  ~~~~~
      threaded:     3.946716  (-19.97%, -0.98)
      ### data-env-vars.name                  = mozilla-central-2024-03-22-zstd-sparse-revlog
      not-threaded: 35.360483  ~~~~~
      threaded:     27.228980  (-23.00%, -8.13)
      ### data-env-vars.name                  = mozilla-unified-2024-03-22-zstd-sparse-revlog
      not-threaded: 35.465007  ~~~~~
      threaded:     28.110867  (-20.74%, -7.35)
      ### data-env-vars.name                  = mozilla-try-2024-03-26-zstd-sparse-revlog
        # benchmark.variants.read-from-memory = no
      not-threaded: 107.609183  ~~~~~
      threaded:     91.002420 (-15.43%, -16.61)
      c124308e3cd4
    • Pierre-Yves David's avatar
      stream-clone-v2: bypass the vfs to write the file on disk · 22e264ac7f60
      Pierre-Yves David authored
      Now that all preparation is done in the main thread, we no longer need the vfs
      to be involved at the writing end.
      
      This very significantly speeds things up.
      
      
      ### benchmark.name                         = hg.perf.exchange.stream.consume
        # bin-env-vars.hg.flavor                 = default
        # bin-env-vars.hg.py-re2-module          = default
        # benchmark.variants.memory-target       = default
        # benchmark.variants.num-writer          = default
        # benchmark.variants.parallel-processing = yes
        # benchmark.variants.progress            = no
        # benchmark.variants.read-from-memory    = yes
        # benchmark.variants.version             = v2
       ## data-env-vars.name                     = mercurial-public-2024-03-22-zstd-sparse-revlog
      prev-change:  0.217293  ~~~~~
      this-change:  0.154647  (-28.83%, -0.06)
       ## data-env-vars.name                     = netbeans-2019-11-07-zstd-sparse-revlog
      prev-change:  11.222771  ~~~~~
      this-change:  7.843554  (-30.11%, -3.38)
       ## data-env-vars.name                     = netbsd-xsrc-all-2024-09-19-zstd-sparse-revlog
      prev-change:  4.465113  ~~~~~
      this-change:  3.040664  (-31.90%, -1.42)
       ## data-env-vars.name                     = netbsd-xsrc-draft-2024-09-19-zstd-sparse-revlog
      prev-change:  4.667360  ~~~~~
      this-change:  3.070976  (-34.20%, -1.60)
       ## data-env-vars.name                     = pypy-2024-03-22-zstd-sparse-revlog
      prev-change:  2.559670  ~~~~~
      this-change:  1.832118  (-28.42%, -0.73)
       ## data-env-vars.name                     = heptapod-public-2024-03-25-zstd-sparse-revlog
      prev-change:  6.123469  ~~~~~
      this-change:  4.478754  (-26.86%, -1.64)
       ## data-env-vars.name                     = mozilla-central-2024-03-22-zstd-sparse-revlog
      prev-change:  44.781498  ~~~~~
      this-change:  30.349379 (-32.23%, -14.43)
       ## data-env-vars.name                     = mozilla-unified-2024-03-22-zstd-sparse-revlog
      prev-change:  44.396959  ~~~~~
      this-change:  31.179906 (-29.77%, -13.22)
       ## data-env-vars.name                     = mozilla-try-2024-03-26-zstd-sparse-revlog
        # benchmark.variants.read-from-memory    = no
      prev-change:  108.552706  ~~~~~
      this-change:  91.454508 (-15.75%, -17.10)
      22e264ac7f60
    • Pierre-Yves David's avatar
      stream-clone-v2: centralize preparation for streamed files · 5b8f6e198a6e
      Pierre-Yves David authored
      The vfs will do some operation to prepare the creation of the streamed file. It
      creates the necessary directories and it adds the file to the fncache. The
      directory creation is much more expensive than it should, and the same
      directory are being checked over and over. And the other part, like "adding the
      file to the fncache" increase the amount of GIL holding time.
      
      So we add a function to the vfs to do such preparation, and we centralise it in
      the "parsing" thread.  This is done so that each directory are only "created" once.
      
      This does not yield speed up yet, because the writer thread still go through the
      same slow logic. This will be dealt with in the next changesets.
      5b8f6e198a6e
  15. Jan 20, 2025
    • Pierre-Yves David's avatar
      stream-clone-v2: disable buffering when writing the files · 7fc882f7fada
      Pierre-Yves David authored
      Not going through python's buffer while writing chunk is not only faster, but it
      reduce the amount of operation that are needed under the GIL, increasing the
      benefit from the threads.
      
      ### benchmark.name                         = hg.perf.exchange.stream.consume
        # bin-env-vars.hg.flavor                 = default
        # bin-env-vars.hg.py-re2-module          = default
        # benchmark.variants.memory-target       = default
        # benchmark.variants.num-writer          = default
        # benchmark.variants.parallel-processing = yes
        # benchmark.variants.progress            = no
        # benchmark.variants.read-from-memory    = yes
        # benchmark.variants.version             = v2
       ## data-env-vars.name                     = mercurial-public-2024-03-22-zstd-sparse-revlog
      prev-change:  0.233780  ~~~~~
      this-change:  0.217293   (-7.05%, -0.02)
       ## data-env-vars.name                     = netbeans-2019-11-07-zstd-sparse-revlog
      prev-change:  11.754377  ~~~~~
      this-change:  11.222771   (-4.52%, -0.53)
       ## data-env-vars.name                     = netbsd-xsrc-all-2024-09-19-zstd-sparse-revlog
      prev-change:  4.735520  ~~~~~
      this-change:  4.465113   (-5.71%, -0.27)
       ## data-env-vars.name                     = netbsd-xsrc-draft-2024-09-19-zstd-sparse-revlog
      prev-change:  4.728870  ~~~~~
      this-change:  4.667360   (-1.30%, -0.06)
       ## data-env-vars.name                     = pypy-2024-03-22-zstd-sparse-revlog
      prev-change:  2.783987  ~~~~~
      this-change:  2.559670   (-8.06%, -0.22)
       ## data-env-vars.name                     = heptapod-public-2024-03-25-zstd-sparse-revlog
      prev-change:  6.532561  ~~~~~
      this-change:  6.123469   (-6.26%, -0.41)
       ## data-env-vars.name                     = mozilla-central-2024-03-22-zstd-sparse-revlog
      prev-change:  47.154728  ~~~~~
      this-change:  44.781498   (-5.03%, -2.37)
       ## data-env-vars.name                     = mozilla-unified-2024-03-22-zstd-sparse-revlog
      prev-change:  47.166569  ~~~~~
      this-change:  44.396959   (-5.87%, -2.77)
       ## data-env-vars.name                     = mozilla-try-2024-03-26-zstd-sparse-revlog
        # benchmark.variants.read-from-memory    = no
      prev-change:  113.696529  ~~~~~
      this-change:  108.552706   (-4.52%, -5.14)
      7fc882f7fada
  16. Nov 30, 2024
  17. Nov 29, 2024
    • Pierre-Yves David's avatar
      stream-clone-v2: disable the garbage collector during consumption · 34fa51c25112
      Pierre-Yves David authored
      As often with code that creates a lot of temporary small container (e.g.
      tuple), disable the gc yield a visible speedup. This this code does not
      create reference cycle this should not result in any increased memory usage.
      
      ### benchmark.name                         = hg.perf.exchange.stream.consume
        # bin-env-vars.hg.flavor                 = default
        # bin-env-vars.hg.py-re2-module          = default
        # benchmark.variants.memory-target       = default
        # benchmark.variants.num-writer          = default
        # benchmark.variants.parallel-processing = yes
        # benchmark.variants.progress            = no
        # benchmark.variants.read-from-memory    = yes
        # benchmark.variants.version             = v2
       ## data-env-vars.name                     = mercurial-public-2024-03-22-zstd-sparse-revlog
      prev-change:  0.234764  ~~~~~
      this-change:  0.233780
       ## data-env-vars.name                     = netbeans-2019-11-07-zstd-sparse-revlog
      prev-change:  12.037305  ~~~~~
      this-change:  11.754377   (-2.35%, -0.28)
       ## data-env-vars.name                     = netbsd-xsrc-all-2024-09-19-zstd-sparse-revlog
      prev-change:  4.763710  ~~~~~
      this-change:  4.735520
       ## data-env-vars.name                     = netbsd-xsrc-draft-2024-09-19-zstd-sparse-revlog
      prev-change:  4.816293  ~~~~~
      this-change:  4.728870   (-1.82%, -0.09)
       ## data-env-vars.name                     = pypy-2024-03-22-zstd-sparse-revlog
      prev-change:  2.801525  ~~~~~
      this-change:  2.783987
       ## data-env-vars.name                     = heptapod-public-2024-03-25-zstd-sparse-revlog
      prev-change:  6.611609  ~~~~~
      this-change:  6.532561   (-1.20%, -0.08)
       ## data-env-vars.name                     = mozilla-central-2024-03-22-zstd-sparse-revlog
      prev-change:  47.733498  ~~~~~
      this-change:  47.154728   (-1.21%, -0.58)
       ## data-env-vars.name                     = mozilla-unified-2024-03-22-zstd-sparse-revlog
      prev-change:  48.270778  ~~~~~
      this-change:  47.166569   (-2.29%, -1.10)
       ## data-env-vars.name                     = mozilla-try-2024-03-26-zstd-sparse-revlog
        # benchmark.variants.read-from-memory    = no
      prev-change:  115.240169  ~~~~~
      this-change:  113.696529   (-1.34%, -1.54)
      34fa51c25112
  18. Feb 05, 2025
    • Pierre-Yves David's avatar
      stream-clone-v2: simplify the handling of small file · 119cddd02b8c
      Pierre-Yves David authored
      We don't need to a use a thread safe queue for file that come in one block. In
      such case we can significantly simplify the code to avoid using a Queue and
      locks for each small files.
      
      This yield a very significant speedup. The threaded version is finally faster
      than our baseline.
      
      ### benchmark.name                         = hg.perf.exchange.stream.consume
        # bin-env-vars.hg.flavor                 = default
        # bin-env-vars.hg.py-re2-module          = default
        # benchmark.variants.memory-target       = default
        # benchmark.variants.num-writer          = default
        # benchmark.variants.parallel-processing = yes
        # benchmark.variants.progress            = no
        # benchmark.variants.read-from-memory    = yes
        # benchmark.variants.version             = v2
       ## data-env-vars.name                     = mercurial-public-2024-03-22-zstd-sparse-revlog
      baseline:     0.249693  ~~~~~
      prev-change:  0.259730   (+4.02%, +0.01)
      this-change:  0.234764   (-5.98%, -0.01)
       ## data-env-vars.name                     = netbeans-2019-11-07-zstd-sparse-revlog
      baseline:     13.136674  ~~~~~
      prev-change:  13.768783   (+4.81%, +0.63)
      this-change:  12.037305   (-8.37%, -1.10)
       ## data-env-vars.name                     = netbsd-xsrc-all-2024-09-19-zstd-sparse-revlog
      baseline:     5.317709  ~~~~~
      prev-change:  5.679052   (+6.80%, +0.36)
      this-change:  4.763710  (-10.42%, -0.55)
       ## data-env-vars.name                     = netbsd-xsrc-draft-2024-09-19-zstd-sparse-revlog
      baseline:     5.398368  ~~~~~
      prev-change:  5.616571   (+4.04%, +0.22)
      this-change:  4.816293  (-10.78%, -0.58)
       ## data-env-vars.name                     = pypy-2024-03-22-zstd-sparse-revlog
      baseline:     3.acbb55  ~~~~~
      prev-change:  3.164637   (+3.13%, +0.10)
      this-change:  2.801525   (-8.70%, -0.27)
       ## data-env-vars.name                     = heptapod-public-2024-03-25-zstd-sparse-revlog
      baseline:     7.244015  ~~~~~
      prev-change:  7.729637   (+6.70%, +0.49)
      this-change:  6.611609   (-8.73%, -0.63)
       ## data-env-vars.name                     = mozilla-central-2024-03-22-zstd-sparse-revlog
      baseline:     51.934795  ~~~~~
      prev-change:  56.567493   (+8.92%, +4.63)
      this-change:  47.733498   (-8.09%, -4.20)
       ## data-env-vars.name                     = mozilla-unified-2024-03-22-zstd-sparse-revlog
      baseline:     52.253858  ~~~~~
      prev-change:  56.093516   (+7.35%, +3.84)
      this-change:  48.270778   (-7.62%, -3.98)
       ## data-env-vars.name                     = mozilla-try-2024-03-26-zstd-sparse-revlog
        # benchmark.variants.read-from-memory    = no
      baseline:     130.584329  ~~~~~
      prev-change:  125.985212   (-3.52%, -4.60)
      this-change:  115.240169 (-11.75%, -15.34)
      119cddd02b8c
    • Pierre-Yves David's avatar
      util: expose the default chunk size for filechunkiter · c92f9edd362a
      Pierre-Yves David authored
      This will help us to decide when to use it or not in the next changeset.
      c92f9edd362a
  19. Feb 03, 2025
  20. Feb 05, 2025
    • Pierre-Yves David's avatar
      stream-clone-v2: use a Queue implementation without a stdlib Condition object · 0af8965b668a
      Pierre-Yves David authored
      The `threading.Condition` object from the stdlib suffer for excessive
      creation/destruction of the underlying locks. This destroy its performance to a
      quite noticeable point.
      
      Recreating the same logic with persistent lock yields an absurdly big
      performance gain. In practice, most of the slowdown introduced by threading is
      recovered.
      
      This benchmark compare the non-threaded baseline, the naive threaded
      implementation with memory limitation, the previous changeset and this one.
      
      ### benchmark.name                         = hg.perf.exchange.stream.consume
        # bin-env-vars.hg.flavor                 = default
        # bin-env-vars.hg.py-re2-module          = default
        # benchmark.variants.memory-target       = default
        # benchmark.variants.num-writer          = default
        # benchmark.variants.parallel-processing = yes
        # benchmark.variants.progress            = no
        # benchmark.variants.read-from-memory    = yes
        # benchmark.variants.version             = v2
       ## data-env-vars.name                     = mercurial-public-2024-03-22-zstd-sparse-revlog
      no-thread:    0.249693  ~~~~~
      naive-thread: 0.305973  (+22.54%, +0.06)
      prev-change:  0.305442  (+22.33%, +0.06)
      this-change:  0.258450   (+3.51%, +0.01)
       ## data-env-vars.name                     = netbeans-2019-11-07-zstd-sparse-revlog
      no-thread:    13.136674  ~~~~~
      naive-thread: 18.467590  (+40.58%, +5.33)
      prev-change:  17.788205  (+35.41%, +4.65)
      this-change:  13.715187   (+4.40%, +0.58)
       ## data-env-vars.name                     = netbsd-xsrc-all-2024-09-19-zstd-sparse-revlog
      no-thread:    5.317709  ~~~~~
      naive-thread: 7.338505  (+38.00%, +2.02)
      prev-change:  7.122798  (+33.94%, +1.81)
      this-change:  5.687069   (+6.95%, +0.37)
       ## data-env-vars.name                     = netbsd-xsrc-draft-2024-09-19-zstd-sparse-revlog
      no-thread:    5.398368  ~~~~~
      naive-thread: 7.333354  (+35.84%, +1.93)
      prev-change:  7.123533  (+31.96%, +1.73)
      this-change:  5.749082   (+6.50%, +0.35)
       ## data-env-vars.name                     = pypy-2024-03-22-zstd-sparse-revlog
      no-thread:    3.acbb55  ~~~~~
      naive-thread: 4.238172  (+38.11%, +1.17)
      prev-change:  4.095587  (+33.47%, +1.03)
      this-change:  3.179653   (+3.62%, +0.11)
       ## data-env-vars.name                     = heptapod-public-2024-03-25-zstd-sparse-revlog
      no-thread:    7.244015  ~~~~~
      naive-thread: 9.901032  (+36.68%, +2.66)
      prev-change:  9.728038  (+34.29%, +2.48)
      this-change:  7.638277   (+5.44%, +0.39)
       ## data-env-vars.name                     = mozilla-central-2024-03-22-zstd-sparse-revlog
      no-thread:    51.934795  ~~~~~
      naive-thread: 78.194540 (+50.56%, +26.26)
      prev-change:  72.574373 (+39.74%, +20.64)
      this-change:  56.267824  (+8.34%, +4.33)
       ## data-env-vars.name                     = mozilla-unified-2024-03-22-zstd-sparse-revlog
      no-thread:    52.253858  ~~~~~
      naive-thread: 77.492938 (+48.30%, +25.24)
      prev-change:  72.845963 (+39.41%, +20.59)
      this-change:  55.368316  (+5.96%, +3.11)
       ## data-env-vars.name                     = mozilla-try-2024-03-26-zstd-sparse-revlog
        # benchmark.variants.read-from-memory    = no
      no-thread:    130.584329  ~~~~~
      naive-thread: 164.366925 (+25.87%, +33.78)
      prev-change:  154.873570 (+18.60%, +24.29)
      this-change:  132.137611  (+1.19%, +1.55)
      0af8965b668a
  21. Feb 03, 2025
    • Pierre-Yves David's avatar
      stream-clone-v2: no longer use the stdlib Condition object · 363914ba328d
      Pierre-Yves David authored
      The `threading.Condition` object from the stdlib suffer for excessive
      creation/destruction of the underlying locks. This destroy its performance to a
      quite noticeable point.
      
      Recreating the same logic with persistent lock yield an absurd performance gain
      (or recovery from the slow down).
      
      Note that this problem also affect the stdlib `queue.Queue` object. This will be dealt
      with in the next changesets.
      
      
      ### benchmark.name                         = hg.perf.exchange.stream.consume
        # bin-env-vars.hg.flavor                 = default
        # bin-env-vars.hg.py-re2-module          = default
        # benchmark.variants.memory-target       = default
        # benchmark.variants.num-writer          = default
        # benchmark.variants.parallel-processing = yes
        # benchmark.variants.progress            = no
        # benchmark.variants.read-from-memory    = yes
        # benchmark.variants.version             = v2
       ## data-env-vars.name                     = mercurial-public-2024-03-22-zstd-sparse-revlog
      no-thread:    0.249693  ~~~~~
      threaded:     0.292601  (+17.18%, +0.04)
      mem-target:   0.305973  (+22.54%, +0.06)
      this-change:  0.305442  (+22.33%, +0.06)
       ## data-env-vars.name                     = heptapod-public-2024-03-25-zstd-sparse-revlog
      no-thread:    7.244015  ~~~~~
      threaded:     9.387256  (+29.59%, +2.14)
      mem-target:   9.901032  (+36.68%, +2.66)
      this-change:  9.728038  (+34.29%, +2.48)
       ## data-env-vars.name                     = netbeans-2019-11-07-zstd-sparse-revlog
      no-thread:    13.136674  ~~~~~
      threaded:     17.008865  (+29.48%, +3.87)
      mem-target:   18.467590  (+40.58%, +5.33)
      this-change:  17.788205  (+35.41%, +4.65)
       ## data-env-vars.name                     = netbsd-xsrc-all-2024-09-19-zstd-sparse-revlog
      no-thread:    5.317709  ~~~~~
      threaded:     7.107141  (+33.65%, +1.79)
      mem-target:   7.338505  (+38.00%, +2.02)
      this-change:  7.122798  (+33.94%, +1.81)
       ## data-env-vars.name                     = netbsd-xsrc-draft-2024-09-19-zstd-sparse-revlog
      no-thread:    5.398368  ~~~~~
      threaded:     7.163505  (+32.70%, +1.77)
      mem-target:   7.333354  (+35.84%, +1.93)
      this-change:  7.123533  (+31.96%, +1.73)
       ## data-env-vars.name                     = pypy-2024-03-22-zstd-sparse-revlog
      no-thread:    3.acbb55  ~~~~~
      threaded:     3.907693  (+27.34%, +0.84)
      mem-target:   4.238172  (+38.11%, +1.17)
      this-change:  4.095587  (+33.47%, +1.03)
       ## data-env-vars.name                     = mozilla-central-2024-03-22-zstd-sparse-revlog
      no-thread:    51.934795  ~~~~~
      threaded:     66.902619 (+28.82%, +14.97)
      mem-target:   78.194540 (+50.56%, +26.26)
      this-change:  72.574373 (+39.74%, +20.64)
       ## data-env-vars.name                     = mozilla-unified-2024-03-22-zstd-sparse-revlog
      no-thread:    52.253858  ~~~~~
      threaded:     66.397609 (+27.07%, +14.14)
      mem-target:   77.492938 (+48.30%, +25.24)
      this-change:  72.845963 (+39.41%, +20.59)
       ## data-env-vars.name                     = mozilla-try-2024-03-26-zstd-sparse-revlog
        # benchmark.variants.read-from-memory    = no
      no-thread:    130.584329  ~~~~~
      threaded:     145.137477 (+11.14%, +14.55)
      mem-target:   164.366925 (+25.87%, +33.78)
      this-change:  154.873570 (+18.60%, +24.29)
      363914ba328d
  22. Jan 27, 2025
    • Pierre-Yves David's avatar
      stream-clone-v2: introduce a way to limit memory usage of the threaded version · aee193b1c784
      Pierre-Yves David authored
      This mechanism limit the read-aheadto avoid uncontrolled memory usage. Note that
      the memory target is not a hard limit and that mercurial will use more memory
      that specified in some cases, but "not too much more".
      
      The current implementation focus on "simplicity" and could be more efficient.
      However this provide a "safe" baseline for the feature. Now that the overall
      shape of the feature is here we can start making it faster.
      
      Here is more benchmark of how everything slows down. The next series will focus
      on optimizing this code path to actually speeds things up.
      
      ### benchmark.name                         = hg.perf.exchange.stream.consume
        # bin-env-vars.hg.flavor                 = default
        # bin-env-vars.hg.py-re2-module          = default
        # benchmark.variants.memory-target       = default
        # benchmark.variants.num-writer          = default
        # benchmark.variants.parallel-processing = yes
        # benchmark.variants.progress            = no
        # benchmark.variants.read-from-memory    = yes
        # benchmark.variants.version             = v2
       ## data-env-vars.name                     = heptapod-public-2024-03-25-zstd-sparse-revlog
      no-thread:    7.244015  ~~~~~
      write-thread: 9.128669  (+26.02%, +1.88)
      read-thread:  9.387256  (+29.59%, +2.14)
      mem-target:   9.901032  (+36.68%, +2.66)
       ## data-env-vars.name                     = mercurial-public-2024-03-22-zstd-sparse-revlog
      no-thread:    0.249693  ~~~~~
      write-thread: 0.275081  (+10.17%, +0.03)
      read-thread:  0.292601  (+17.18%, +0.04)
      mem-target:   0.305973  (+22.54%, +0.06)
       ## data-env-vars.name                     = netbeans-2019-11-07-zstd-sparse-revlog
      no-thread:    13.136674  ~~~~~
      write-thread: 16.374306  (+24.65%, +3.24)
      read-thread:  17.008865  (+29.48%, +3.87)
      mem-target:   18.467590  (+40.58%, +5.33)
       ## data-env-vars.name                     = netbsd-xsrc-all-2024-09-19-zstd-sparse-revlog
      no-thread:    5.317709  ~~~~~
      write-thread: 6.783031  (+27.56%, +1.47)
      read-thread:  7.107141  (+33.65%, +1.79)
      mem-target:   7.338505  (+38.00%, +2.02)
       ## data-env-vars.name                     = netbsd-xsrc-draft-2024-09-19-zstd-sparse-revlog
      no-thread:    5.398368  ~~~~~
      write-thread: 6.737864  (+24.81%, +1.34)
      read-thread:  7.163505  (+32.70%, +1.77)
      mem-target:   7.333354  (+35.84%, +1.93)
       ## data-env-vars.name                     = pypy-2024-03-22-zstd-sparse-revlog
      no-thread:    3.acbb55  ~~~~~
      write-thread: 3.758324  (+22.48%, +0.69)
      read-thread:  3.907693  (+27.34%, +0.84)
      mem-target:   4.238172  (+38.11%, +1.17)
      
       ## data-env-vars.name                     = mozilla-central-2024-03-22-zstd-sparse-revlog
      no-thread:    51.934795  ~~~~~
      write-thread: 66.561340 (+28.16%, +14.63)
      read-thread:  66.902619 (+28.82%, +14.97)
      mem-target:   78.194540 (+50.56%, +26.26)
       ## data-env-vars.name                     = mozilla-unified-2024-03-22-zstd-sparse-revlog
        # benchmark.variants.read-from-memory    = yes
      no-thread:    52.253858  ~~~~~
      write-thread: 66.955037 (+28.13%, +14.70)
      read-thread:  66.397609 (+27.07%, +14.14)
      mem-target:   77.492938 (+48.30%, +25.24)
       ## data-env-vars.name                     = mozilla-try-2024-03-26-zstd-sparse-revlog
        # benchmark.variants.read-from-memory    = no
      no-thread:    130.584329  ~~~~~
      write-thread: 138.770454   (+6.27%, +8.19)
      read-thread:  145.137477 (+11.14%, +14.55)
      mem-target:   164.366925 (+25.87%, +33.78)
      aee193b1c784
  23. Jan 29, 2025
Loading