From 4e9e79454f7440b94ee729bd87a4fcad8e0db2e2 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 28 May 2023 13:31:05 +0200 Subject: [PATCH] Fix broken transformation functions in the PagingArray classes. --- .../marginalia/array/page/PagingIntArray.java | 57 +++++++++- .../array/page/PagingLongArray.java | 61 ++++++++++- .../algo/IntArrayTransformations2Test.java | 100 ++++++++++++++++++ .../array/algo/LongArraySortNTest.java | 32 +++--- .../array/algo/LongArraySortTest.java | 36 ++++++- .../algo/LongArrayTransformations2Test.java | 97 +++++++++++++++++ 6 files changed, 356 insertions(+), 27 deletions(-) create mode 100644 code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformations2Test.java create mode 100644 code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformations2Test.java diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/page/PagingIntArray.java b/code/libraries/array/src/main/java/nu/marginalia/array/page/PagingIntArray.java index 2d4a13f5..0be2801c 100644 --- a/code/libraries/array/src/main/java/nu/marginalia/array/page/PagingIntArray.java +++ b/code/libraries/array/src/main/java/nu/marginalia/array/page/PagingIntArray.java @@ -163,7 +163,24 @@ public class PagingIntArray extends AbstractPagingArray @Override public void forEach(long start, long end, LongIntConsumer consumer) { - delegateToEachPage(start, end, (page, s, e) -> page.forEach(s, e, consumer)); + assert end >= start; + + int page = partitioningScheme.getPage(start); + + long endPos; + + for (long pos = start; pos < end; pos = endPos) { + endPos = partitioningScheme.getPageEnd(pos, end); + + int sOff = partitioningScheme.getOffset(pos); + int eOff = partitioningScheme.getEndOffset(start, endPos); + + var p = pages[page++]; + + for (long i = sOff; i < eOff; i++) { + consumer.accept(i + (pos - sOff), p.get(i)); + } + } } @Override @@ -182,12 +199,46 @@ public class PagingIntArray extends AbstractPagingArray @Override public void transformEach(long start, long end, IntTransformer transformer) { - delegateToEachPage(start, end, (page, s, e) -> page.transformEach(s, e, transformer)); + assert end >= start; + + int page = partitioningScheme.getPage(start); + + long endPos; + + for (long pos = start; pos < end; pos = endPos) { + endPos = partitioningScheme.getPageEnd(pos, end); + + int sOff = partitioningScheme.getOffset(pos); + int eOff = partitioningScheme.getEndOffset(start, endPos); + + var p = pages[page++]; + + for (long i = sOff; i < eOff; i++) { + p.set(i, transformer.transform(i + (pos - sOff), p.get(i))); + } + } } @Override public void transformEachIO(long start, long end, IntIOTransformer transformer) throws IOException { - delegateToEachPageIO(start, end, (page, s, e) -> page.transformEachIO(s, e, transformer)); + assert end >= start; + + int page = partitioningScheme.getPage(start); + + long endPos; + + for (long pos = start; pos < end; pos = endPos) { + endPos = partitioningScheme.getPageEnd(pos, end); + + int sOff = partitioningScheme.getOffset(pos); + int eOff = partitioningScheme.getEndOffset(start, endPos); + + var p = pages[page++]; + + for (long i = sOff; i < eOff; i++) { + p.set(i, transformer.transform(i + (pos - sOff), p.get(i))); + } + } } @Override diff --git a/code/libraries/array/src/main/java/nu/marginalia/array/page/PagingLongArray.java b/code/libraries/array/src/main/java/nu/marginalia/array/page/PagingLongArray.java index a06ee878..0e01f1f3 100644 --- a/code/libraries/array/src/main/java/nu/marginalia/array/page/PagingLongArray.java +++ b/code/libraries/array/src/main/java/nu/marginalia/array/page/PagingLongArray.java @@ -100,7 +100,7 @@ public class PagingLongArray extends AbstractPagingArray (" + page + ":" + offset + ")"); @@ -156,8 +156,25 @@ public class PagingLongArray extends AbstractPagingArray page.forEach(s, e, transformer)); + public void forEach(long start, long end, LongLongConsumer consumer) { + assert end >= start; + + int page = partitioningScheme.getPage(start); + + long endPos; + + for (long pos = start; pos < end; pos = endPos) { + endPos = partitioningScheme.getPageEnd(pos, end); + + int sOff = partitioningScheme.getOffset(pos); + int eOff = partitioningScheme.getEndOffset(start, endPos); + + var p = pages[page++]; + + for (long i = sOff; i < eOff; i++) { + consumer.accept(i + (pos - sOff), p.get(i)); + } + } } @Override @@ -176,12 +193,46 @@ public class PagingLongArray extends AbstractPagingArray page.transformEach(s, e, transformer)); + assert end >= start; + + int page = partitioningScheme.getPage(start); + + long endPos; + + for (long pos = start; pos < end; pos = endPos) { + endPos = partitioningScheme.getPageEnd(pos, end); + + int sOff = partitioningScheme.getOffset(pos); + int eOff = partitioningScheme.getEndOffset(start, endPos); + + var p = pages[page++]; + + for (long i = sOff; i < eOff; i++) { + p.set(i, transformer.transform(i + (pos - sOff), p.get(i))); + } + } } @Override public void transformEachIO(long start, long end, LongIOTransformer transformer) throws IOException { - delegateToEachPageIO(start, end, (page, s, e) -> page.transformEachIO(s, e, transformer)); + assert end >= start; + + int page = partitioningScheme.getPage(start); + + long endPos; + + for (long pos = start; pos < end; pos = endPos) { + endPos = partitioningScheme.getPageEnd(pos, end); + + int sOff = partitioningScheme.getOffset(pos); + int eOff = partitioningScheme.getEndOffset(start, endPos); + + var p = pages[page++]; + + for (long i = sOff; i < eOff; i++) { + p.set(i, transformer.transform(i + (pos - sOff), p.get(i))); + } + } } @Override diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformations2Test.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformations2Test.java new file mode 100644 index 00000000..ce7298d4 --- /dev/null +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformations2Test.java @@ -0,0 +1,100 @@ +package nu.marginalia.array.algo; + +import nu.marginalia.array.IntArray; +import nu.marginalia.array.LongArray; +import nu.marginalia.array.page.IntArrayPage; +import nu.marginalia.array.page.LongArrayPage; +import nu.marginalia.array.page.PagingIntArray; +import nu.marginalia.array.page.PagingLongArray; +import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class IntArrayTransformations2Test { + IntArray basic; + IntArray paged; + IntArray shifted; + + final int size = 1026; + + @BeforeEach + public void setUp() { + basic = IntArrayPage.onHeap(size); + paged = PagingIntArray.newOnHeap(new PowerOf2PartitioningScheme(32), size); + shifted = IntArrayPage.onHeap(size + 30).shifted(30); + + int[] vals = new int[size]; + for (int i = 0; i < vals.length; i++) { + vals[i] = i+10; + } + basic.set(0, vals); + paged.set(0, vals); + shifted.set(0, vals); + } + @Test + void forEach() { + foreachTester(basic); + foreachTester(paged); + foreachTester(shifted); + } + @Test + void transformEach() { + transformTester(basic); + transformTester(paged); + transformTester(shifted); + } + + @Test + void transformEachIO() throws IOException { + transformTesterIO(basic); + transformTesterIO(paged); + transformTesterIO(shifted); + } + + private void transformTester(IntArray array) { + ArrayList offsets = new ArrayList<>(); + + array.transformEach(0, size, (i, val) -> { + assertEquals(i+10, val); + offsets.add(i); + return -val; + }); + for (int i = 0; i < size; i++) { + assertEquals(-(i+10), array.get(i)); + } + } + + private void transformTesterIO(IntArray array) throws IOException { + ArrayList offsets = new ArrayList<>(); + + array.transformEachIO(0, size, (i, val) -> { + assertEquals(i+10, val); + offsets.add(i); + return -val; + }); + + for (int i = 0; i < size; i++) { + assertEquals(-(i+10), array.get(i)); + } + + for (int i = 0; i < size; i++) { + assertEquals(offsets.get(i), i); + } + } + + private void foreachTester(IntArray array) { + ArrayList offsets = new ArrayList<>(); + array.forEach(0, size, (i, val) -> { + assertEquals(i+10, val); + offsets.add(i); + }); + for (int i = 0; i < size; i++) { + assertEquals(offsets.get(i), i); + } + } +} \ No newline at end of file diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortNTest.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortNTest.java index af4c9328..883e9b65 100644 --- a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortNTest.java +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortNTest.java @@ -27,9 +27,7 @@ class LongArraySortNTest { LongArray paged; LongArray shifted; - Long2ObjectOpenHashMap basicPairs; - Long2ObjectOpenHashMap pagedPairs; - Long2ObjectOpenHashMap shiftedPairs; + Long2ObjectOpenHashMap dataAsPairs; final int size = 1026; @@ -48,13 +46,11 @@ class LongArraySortNTest { values[i] = -values[i]; } - basic.transformEach(0, size, (i, old) -> values[(int) i]); - paged.transformEach(0, size, (i, old) -> values[(int) i]); - shifted.transformEach(0, size, (i, old) -> values[(int) i]); + basic.set(0, values); + paged.set(0, values); + shifted.set(0, values); - basicPairs = asPairs(basic); - pagedPairs = asPairs(paged); - shiftedPairs = asPairs(shifted); + dataAsPairs = asPairs(basic); } @@ -144,9 +140,9 @@ class LongArraySortNTest { shifted.insertionSortN(2, 0, size); assertTrue(shifted.isSortedN(2, 0, size)); - compare(basic, basicPairs); - compare(paged, pagedPairs); - compare(shifted, shiftedPairs); + compare(basic, dataAsPairs); + compare(paged, dataAsPairs); + compare(shifted, dataAsPairs); } @Test @@ -160,9 +156,9 @@ class LongArraySortNTest { shifted.quickSortN(2, 0, size); assertTrue(shifted.isSortedN(2, 0, size)); - compare(basic, basicPairs); - compare(paged, pagedPairs); - compare(shifted, shiftedPairs); + compare(basic, dataAsPairs); + compare(paged, dataAsPairs); + compare(shifted, dataAsPairs); } @Test @@ -177,9 +173,9 @@ class LongArraySortNTest { shifted.mergeSortN(2, 0, size, Path.of("/tmp")); assertTrue(shifted.isSortedN(2, 0, size)); - compare(basic, basicPairs); - compare(paged, pagedPairs); - compare(shifted, shiftedPairs); + compare(basic, dataAsPairs); + compare(paged, dataAsPairs); + compare(shifted, dataAsPairs); } private Long2ObjectOpenHashMap asPairs(LongArray array) { diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortTest.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortTest.java index 535b7671..fc532cf3 100644 --- a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortTest.java +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortTest.java @@ -1,12 +1,12 @@ package nu.marginalia.array.algo; +import it.unimi.dsi.fastutil.longs.LongOpenHashSet; import nu.marginalia.array.LongArray; import nu.marginalia.array.page.LongArrayPage; import nu.marginalia.array.page.PagingLongArray; import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; import nu.marginalia.util.test.TestUtil; import org.apache.commons.lang3.ArrayUtils; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -25,6 +25,9 @@ class LongArraySortTest { LongArray basic; LongArray paged; LongArray shifted; + + LongOpenHashSet valueSet; + final int size = 1026; @BeforeEach @@ -33,12 +36,19 @@ class LongArraySortTest { paged = PagingLongArray.newOnHeap(new PowerOf2PartitioningScheme(32), size); shifted = LongArrayPage.onHeap(size + 30).shifted(30); + valueSet = new LongOpenHashSet(); + var random = new Random(); long[] values = new long[size]; for (int i = 0; i < size; i++) { values[i] = random.nextInt(0, 1000); + valueSet.add(values[i]); } + basic.set(0, values); + paged.set(0, values); + shifted.set(0, values); + basic.transformEach(0, size, (i, old) -> values[(int) i]); paged.transformEach(0, size, (i, old) -> values[(int) i]); shifted.transformEach(0, size, (i, old) -> values[(int) i]); @@ -114,6 +124,16 @@ class LongArraySortTest { } + void verifyValuesPresent(LongArray array) { + LongOpenHashSet actualValues = new LongOpenHashSet(); + + for (int i = 0; i < array.size(); i++) { + actualValues.add(array.get(i)); + } + + assertEquals(valueSet, actualValues); + } + @Test void insertionSort() { basic.insertionSort(0, size); @@ -124,6 +144,10 @@ class LongArraySortTest { shifted.insertionSort(0, size); assertTrue(shifted.isSorted(0, 128)); + + verifyValuesPresent(basic); + verifyValuesPresent(paged); + verifyValuesPresent(shifted); } @Test @@ -136,6 +160,12 @@ class LongArraySortTest { shifted.quickSort(0, size); assertTrue(shifted.isSorted(0, size)); + + + verifyValuesPresent(basic); + verifyValuesPresent(paged); + verifyValuesPresent(shifted); + } @Test @@ -148,6 +178,10 @@ class LongArraySortTest { shifted.mergeSort(0, size, Path.of("/tmp")); assertTrue(shifted.isSorted(0, size)); + + verifyValuesPresent(basic); + verifyValuesPresent(paged); + verifyValuesPresent(shifted); } @Test diff --git a/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformations2Test.java b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformations2Test.java new file mode 100644 index 00000000..4045049a --- /dev/null +++ b/code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformations2Test.java @@ -0,0 +1,97 @@ +package nu.marginalia.array.algo; + +import nu.marginalia.array.LongArray; +import nu.marginalia.array.page.LongArrayPage; +import nu.marginalia.array.page.PagingLongArray; +import nu.marginalia.array.scheme.PowerOf2PartitioningScheme; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class LongArrayTransformations2Test { + LongArray basic; + LongArray paged; + LongArray shifted; + + final int size = 1026; + + @BeforeEach + public void setUp() { + basic = LongArrayPage.onHeap(size); + paged = PagingLongArray.newOnHeap(new PowerOf2PartitioningScheme(32), size); + shifted = LongArrayPage.onHeap(size + 30).shifted(30); + + long[] vals = new long[size]; + for (int i = 0; i < vals.length; i++) { + vals[i] = i+10; + } + basic.set(0, vals); + paged.set(0, vals); + shifted.set(0, vals); + } + @Test + void forEach() { + foreachTester(basic); + foreachTester(paged); + foreachTester(shifted); + } + @Test + void transformEach() { + transformTester(basic); + transformTester(paged); + transformTester(shifted); + } + + @Test + void transformEachIO() throws IOException { + transformTesterIO(basic); + transformTesterIO(paged); + transformTesterIO(shifted); + } + + private void transformTester(LongArray array) { + ArrayList offsets = new ArrayList<>(); + + array.transformEach(0, size, (i, val) -> { + assertEquals(i+10, val); + offsets.add(i); + return -val; + }); + for (int i = 0; i < size; i++) { + assertEquals(-(i+10), array.get(i)); + } + } + + private void transformTesterIO(LongArray array) throws IOException { + ArrayList offsets = new ArrayList<>(); + + array.transformEachIO(0, size, (i, val) -> { + assertEquals(i+10, val); + offsets.add(i); + return -val; + }); + + for (int i = 0; i < size; i++) { + assertEquals(-(i+10), array.get(i)); + } + + for (int i = 0; i < size; i++) { + assertEquals(offsets.get(i), i); + } + } + + private void foreachTester(LongArray array) { + ArrayList offsets = new ArrayList<>(); + array.forEach(0, size, (i, val) -> { + assertEquals(i+10, val); + offsets.add(i); + }); + for (int i = 0; i < size; i++) { + assertEquals(offsets.get(i), i); + } + } +} \ No newline at end of file