Bisc labelled perms

CHANGELOG.md

@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+### Added
+- `LabelledPerms` in `permuta.bisc`: labels of permutations kept as one bitmap
+  per length (lexicographic order), made from a property, given as bitmaps, or both.
+  `auto_bisc` and `bisc` accept it as input.
+- `auto_bisc(prop, labels="bitmaps")`: keeps a property's labels as bitmaps instead
+  of lists of permutations, for the same result in far less memory at longer
+  lengths. `labels="lists"`, the default, is the previous behaviour.
+
 ### Fixed
 - Fixed bug in `Perm.rtlmax_ltrmin_decomposition` and fixed associated tests.
 

README.rst

@@ -516,7 +516,24 @@ permutations that can be imported from ``permuta.bisc.perm_properties``, such
 as ``smooth``, ``forest-like``, ``baxter``, ``simsun``, ``quick_sortable``, etc.
 
 Both ``bisc`` and ``auto_bisc`` can accept input in the form of a property,
-or a list of permutations (satisfying some property).
+a list of permutations (satisfying some property), or a ``LabelledPerms``.
+
+``auto_bisc`` keeps the permutations satisfying a property, and those that do
+not, as lists. For long runs it can keep them as one bitmap per length instead,
+with the same result in far less memory:
+``auto_bisc(Perm.stack_sortable, labels="bitmaps")``. Such labels can also be
+given directly as a ``LabelledPerms``, made from a property, from bitmaps, or
+both, and ``bisc`` accepts it too.
+
+.. code-block:: python
+
+    >>> labels = LabelledPerms(Perm.stack_sortable)
+    >>> for n in range(6):
+    ...     labels.add_length(n)
+    >>> len(labels.good[5]), len(labels.bad[5])
+    (42, 78)
+    >>> bisc(labels, 3, 5)
+    {3: {Perm((1, 2, 0)): [set()]}}
 
 License
 #######

permuta/bisc/__init__.py

@@ -1,6 +1,13 @@
 # type: ignore
 
-from .bisc import auto_bisc, bisc, create_bisc_input, read_bisc_file, write_bisc_files
+from .bisc import (
+    LabelledPerms,
+    auto_bisc,
+    bisc,
+    create_bisc_input,
+    read_bisc_file,
+    write_bisc_files,
+)
 from .bisc_subfunctions import (
     patterns_suffice_for_bad,
     patterns_suffice_for_good,
@@ -10,6 +17,7 @@
 )
 
 __all__ = [
+    "LabelledPerms",
     "auto_bisc",
     "bisc",
     "create_bisc_input",

permuta/bisc/bisc.py

@@ -4,6 +4,7 @@
 import os
 import types
 from collections import defaultdict
+from math import factorial
 
 from permuta.bisc.bisc_subfunctions import (
     forb,
@@ -36,12 +37,19 @@ def bisc(A, m, n=None, report=False):
     elif isinstance(A, dict):
         D = A
 
+    elif isinstance(A, LabelledPerms):
+        D = A.good
+
+    elif isinstance(A, _LengthViews):
+        D = A
+
     else:
         print(
             """BiSC can accept inputs:
         1. A list of permutations
         2. A property of permutations
-        3. A dictionary of lengths pointing to permutations"""
+        3. A dictionary of lengths pointing to permutations
+        4. A LabelledPerms (labels kept as bitmaps)"""
         )
         assert False
 
@@ -54,7 +62,20 @@ def bisc(A, m, n=None, report=False):
     return SG
 
 
-def auto_bisc(prop):
+def auto_bisc(prop, labels="lists"):
+    """
+    labels says how the labels of a property (a function) are kept:
+    "lists" (the default, as before): the good and the bad permutations of
+    each length as lists of Perm objects;
+    "bitmaps": one bitmap per length (LabelledPerms), for the same result in
+    far less memory, at the cost of generating a length's permutations again
+    each time it is read.
+    A LabelledPerms can also be given as prop directly.
+    """
+    if labels not in ("lists", "bitmaps"):
+        raise ValueError('labels must be "lists" or "bitmaps"')
+    if labels == "bitmaps" and isinstance(prop, types.FunctionType):
+        prop = LabelledPerms(prop)
     L = 8  # Want to sanity check on at least S8, and one above n
     n = 4
     m = 2
@@ -88,6 +109,20 @@ def auto_bisc(prop):
                 else:
                     B[i].append(perm)
 
+    elif isinstance(prop, LabelledPerms):
+        # Labels kept as one bitmap per length: A and B are views of the good
+        # and the bad permutations, generated length by length when they are
+        # read, instead of lists. Lengths not given are labelled with the
+        # LabelledPerms's property, if it has one
+        labelled = prop
+        for i in range(L + 1):
+            if i not in labelled.bits:
+                if labelled.prop is None:
+                    print("You should have labels up to length at least 8")
+                    return
+                labelled.add_length(i)
+        A, B = labelled.good, labelled.bad
+
     elif (
         isinstance(prop, tuple)
         and isinstance(prop[0], dict)
@@ -229,6 +264,15 @@ def auto_bisc(prop):
                         else:
                             B[i].append(perm)
 
+            elif isinstance(prop, LabelledPerms):
+                for i in range(oldL + 1, L + 1):
+                    if i not in labelled.bits:
+                        if labelled.prop is None:
+                            print("You need to add labels of longer permutations")
+                            return
+                        print("Adding perms of length {}".format(i))
+                        labelled.add_length(i)
+
             elif isinstance(prop, tuple) and L > min(max(A.keys()), max(B.keys())):
                 print("You need to add longer permutations to the dictionaries")
                 return
@@ -269,6 +313,93 @@ def auto_bisc(prop):
                     return
 
 
+class LabelledPerms:
+    """Labels of permutations, one bitmap per length, as input to auto_bisc
+    and bisc: which permutations of each length satisfy a property (good) and
+    which do not (bad).
+
+    Bit i of length n is the label of Perm.unrank(i, n), in byte i // 8, from
+    the low bit; so a permutation perm of length n is at bit
+    perm.rank() - sum(factorial(k) for k in range(n)). This is the order in
+    which Perm.of_length(n) yields the permutations (lexicographic), which the
+    views use to iterate. The bitmaps can be given (bitmaps={n: bytes}), made
+    from a property (add_length), or both: auto_bisc labels the lengths it
+    needs and is not given with prop, and stops, as for a tuple of
+    dictionaries, when there is no prop.
+
+    auto_bisc reads its good and bad permutations only length by length:
+    iterating over a length, counting it, asking whether a length is there.
+    The views good and bad do that from the bitmaps, generating a length's
+    permutations again each time it is iterated, in the same order as the
+    lists a property input makes. A bitmap of S_10 is 454 KB; S_10 as a list
+    of Perm objects is about 0.9 GB.
+    """
+
+    def __init__(self, prop=None, bitmaps=None):
+        self.prop = prop
+        self.bits = {}
+        self.counts = {}
+        for n, data in (bitmaps or {}).items():
+            data = bytes(data)
+            if len(data) != (factorial(n) + 7) // 8:
+                raise ValueError("A bitmap of length {} has the wrong size".format(n))
+            self.bits[n] = data
+            self.counts[n] = sum(bin(b).count("1") for b in data)
+        self.good = _LengthViews(self, True)
+        self.bad = _LengthViews(self, False)
+
+    def add_length(self, n):
+        bits = bytearray((factorial(n) + 7) // 8)
+        count = 0
+        for i, perm in enumerate(Perm.of_length(n)):
+            if self.prop(perm):
+                bits[i >> 3] |= 1 << (i & 7)
+                count += 1
+        self.bits[n] = bits
+        self.counts[n] = count
+
+
+class _LengthViews:
+    """length -> the good (or bad) permutations of that length, read-only, as
+    the dictionaries of lists auto_bisc builds from the other inputs."""
+
+    def __init__(self, labelled, good):
+        self.labelled = labelled
+        self.is_good = good
+
+    def keys(self):
+        return self.labelled.bits.keys()
+
+    def __contains__(self, n):
+        return n in self.labelled.bits
+
+    def __getitem__(self, n):
+        if n not in self.labelled.bits:
+            raise KeyError(n)
+        return _PermsOfLength(self.labelled, n, self.is_good)
+
+
+class _PermsOfLength:
+    def __init__(self, labelled, n, good):
+        self.labelled = labelled
+        self.n = n
+        self.want = 1 if good else 0
+
+    def __iter__(self):
+        bits = self.labelled.bits[self.n]
+        want = self.want
+        for i, perm in enumerate(Perm.of_length(self.n)):
+            if (bits[i >> 3] >> (i & 7)) & 1 == want:
+                yield perm
+
+    def __len__(self):
+        good = self.labelled.counts[self.n]
+        return good if self.want else factorial(self.n) - good
+
+    def __bool__(self):
+        return len(self) > 0
+
+
 def create_bisc_input(N, prop):
     """
     Create a dictionary, D, containing keys 1, 2, 3, ..., N. Each key points to

tests/bisc/test_labelled_perms.py

@@ -0,0 +1,139 @@
+import contextlib
+import io
+from math import factorial
+
+import pytest
+
+from permuta.bisc.bisc import LabelledPerms, auto_bisc, bisc
+from permuta.patterns.perm import Perm
+
+
+def stack_sortable(perm):
+    return perm.avoids(Perm((1, 2, 0)))
+
+
+def labelled(prop, top):
+    labels = LabelledPerms(prop)
+    for n in range(top + 1):
+        labels.add_length(n)
+    return labels
+
+
+def test_views_match_lists():
+    labels = labelled(stack_sortable, 6)
+    for n in range(7):
+        good = [perm for perm in Perm.of_length(n) if stack_sortable(perm)]
+        bad = [perm for perm in Perm.of_length(n) if not stack_sortable(perm)]
+        assert list(labels.good[n]) == good
+        assert list(labels.bad[n]) == bad
+        assert len(labels.good[n]) == len(good)
+        assert len(labels.bad[n]) == len(bad)
+        assert bool(labels.bad[n]) == bool(bad)
+    assert sorted(labels.good.keys()) == list(range(7))
+    assert 6 in labels.bad and 7 not in labels.bad
+    with pytest.raises(KeyError):
+        _ = labels.good[7]
+
+
+def test_bit_order_is_perm_unrank():
+    # Bit i of length n is the label of Perm.unrank(i, n), and the permutation
+    # perm is at bit perm.rank() minus the number of shorter permutations
+    labels = labelled(stack_sortable, 6)
+    shorter = 0
+    for n in range(7):
+        bits = labels.bits[n]
+        for i in range(factorial(n)):
+            perm = Perm.unrank(i, n)
+            assert perm.rank() - shorter == i
+            assert bool(bits[i >> 3] >> (i & 7) & 1) == stack_sortable(perm)
+        shorter += factorial(n)
+
+
+def test_bitmaps_round_trip():
+    labels = labelled(stack_sortable, 6)
+    again = LabelledPerms(bitmaps=labels.bits)
+    for n in range(7):
+        assert list(again.good[n]) == list(labels.good[n])
+        assert list(again.bad[n]) == list(labels.bad[n])
+    with pytest.raises(ValueError):
+        LabelledPerms(bitmaps={3: b"\x00\x00"})  # 3! bits fit in one byte
+
+
+def test_bisc_accepts_labelled_perms():
+    expected = {3: {Perm((1, 2, 0)): [set()]}}
+    assert bisc(stack_sortable, 3, 5) == expected
+    assert bisc(labelled(stack_sortable, 5), 3, 5) == expected
+
+
+def test_auto_bisc_labels_option():
+    expected = {3: {Perm((1, 2, 0)): [set()]}}
+    assert auto_bisc(stack_sortable) == expected
+    assert auto_bisc(stack_sortable, labels="lists") == expected
+    assert auto_bisc(stack_sortable, labels="bitmaps") == expected
+    with pytest.raises(ValueError):
+        auto_bisc(stack_sortable, labels="sets")
+
+
+def test_auto_bisc_labelled_perms_input():
+    expected = {3: {Perm((1, 2, 0)): [set()]}}
+    assert auto_bisc(labelled(stack_sortable, 8)) == expected
+    # Given bitmaps and no property: the lengths auto_bisc needs must be there
+    bits = labelled(stack_sortable, 8).bits
+    assert auto_bisc(LabelledPerms(bitmaps=bits)) == expected
+    short = {n: bits[n] for n in range(6)}
+    assert auto_bisc(LabelledPerms(bitmaps=short)) is None
+
+
+def test_west_2_stack_sortable_same_as_lists():
+    def west_2(perm):
+        def stack_sort(word):
+            out, stack = [], []
+            for x in word:
+                while stack and stack[-1] < x:
+                    out.append(stack.pop())
+                stack.append(x)
+            return out + stack[::-1]
+
+        return stack_sort(stack_sort(list(perm))) == sorted(perm)
+
+    assert auto_bisc(west_2, labels="bitmaps") == auto_bisc(west_2)
+
+
+def derangement_in_av_213_231_312(perm):
+    """One member at each even length, none at odd: auto_bisc keeps learning longer
+    patterns while each step stays cheap, so it soon needs more labels."""
+    return perm.avoids(Perm((1, 0, 2)), Perm((1, 2, 0)), Perm((2, 0, 1))) and all(
+        value != index for index, value in enumerate(perm)
+    )
+
+
+def test_auto_bisc_stops_without_labels_it_needs():
+    # Bitmaps up to length 8 and no property: auto_bisc needs length 9 and stops
+    bits = labelled(derangement_in_av_213_231_312, 8).bits
+    output = io.StringIO()
+    with contextlib.redirect_stdout(output):
+        assert auto_bisc(LabelledPerms(bitmaps=bits)) is None
+    assert "You need to add labels of longer permutations" in output.getvalue()
+
+
+class ReachedLength9(Exception):
+    pass
+
+
+def test_auto_bisc_labels_new_lengths_from_the_property():
+    # When auto_bisc grows past the labelled lengths, it labels the new ones with
+    # the property; stopped at the first permutation of length 9, both approaches
+    # have printed the same
+    def prop(perm):
+        if len(perm) == 9:
+            raise ReachedLength9
+        return derangement_in_av_213_231_312(perm)
+
+    outputs = []
+    for labels in ("lists", "bitmaps"):
+        output = io.StringIO()
+        with contextlib.redirect_stdout(output), pytest.raises(ReachedLength9):
+            auto_bisc(prop, labels=labels)
+        outputs.append(output.getvalue())
+    assert outputs[0] == outputs[1]
+    assert "Adding perms of length 9" in outputs[1]