161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix broken transformation functions in the PagingArray classes.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-05-28 13:31:05 +02:00
parent b0bc07b4e7
commit 4e9e79454f
6 changed files with 356 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
code/libraries/array/src/main/java/nu/marginalia/array/page/PagingIntArray.java
View File

@ -163,7 +163,24 @@ public class PagingIntArray extends AbstractPagingArray<IntArrayPage, IntBuffer>
@Override
public void forEach(long start, long end, LongIntConsumer consumer) {
delegateToEachPage(start, end, (page, s, e) -> page.forEach(s, e, consumer));
assert end >= start;
int page = partitioningScheme.getPage(start);
long endPos;
for (long pos = start; pos < end; pos = endPos) {
endPos = partitioningScheme.getPageEnd(pos, end);
int sOff = partitioningScheme.getOffset(pos);
int eOff = partitioningScheme.getEndOffset(start, endPos);
var p = pages[page++];
for (long i = sOff; i < eOff; i++) {
consumer.accept(i + (pos - sOff), p.get(i));
}
}
}
@Override
@ -182,12 +199,46 @@ public class PagingIntArray extends AbstractPagingArray<IntArrayPage, IntBuffer>
@Override
public void transformEach(long start, long end, IntTransformer transformer) {
delegateToEachPage(start, end, (page, s, e) -> page.transformEach(s, e, transformer));
assert end >= start;
int page = partitioningScheme.getPage(start);
long endPos;
for (long pos = start; pos < end; pos = endPos) {
endPos = partitioningScheme.getPageEnd(pos, end);
int sOff = partitioningScheme.getOffset(pos);
int eOff = partitioningScheme.getEndOffset(start, endPos);
var p = pages[page++];
for (long i = sOff; i < eOff; i++) {
p.set(i, transformer.transform(i + (pos - sOff), p.get(i)));
}
}
}
@Override
public void transformEachIO(long start, long end, IntIOTransformer transformer) throws IOException {
delegateToEachPageIO(start, end, (page, s, e) -> page.transformEachIO(s, e, transformer));
assert end >= start;
int page = partitioningScheme.getPage(start);
long endPos;
for (long pos = start; pos < end; pos = endPos) {
endPos = partitioningScheme.getPageEnd(pos, end);
int sOff = partitioningScheme.getOffset(pos);
int eOff = partitioningScheme.getEndOffset(start, endPos);
var p = pages[page++];
for (long i = sOff; i < eOff; i++) {
p.set(i, transformer.transform(i + (pos - sOff), p.get(i)));
}
}
}
@Override

61
code/libraries/array/src/main/java/nu/marginalia/array/page/PagingLongArray.java
View File

@ -100,7 +100,7 @@ public class PagingLongArray extends AbstractPagingArray<LongArrayPage, LongBuff
int offset = partitioningScheme.getOffset(pos);
try {
return pages[page].get(partitioningScheme.getOffset(pos));
return pages[page].get(offset);
}
catch (IndexOutOfBoundsException ex) {
throw new IndexOutOfBoundsException("Index out of bounds for " + pos + " => (" + page + ":" + offset + ")");
@ -156,8 +156,25 @@ public class PagingLongArray extends AbstractPagingArray<LongArrayPage, LongBuff
}
@Override
public void forEach(long start, long end, LongLongConsumer transformer) {
delegateToEachPage(start, end, (page, s, e) -> page.forEach(s, e, transformer));
public void forEach(long start, long end, LongLongConsumer consumer) {
assert end >= start;
int page = partitioningScheme.getPage(start);
long endPos;
for (long pos = start; pos < end; pos = endPos) {
endPos = partitioningScheme.getPageEnd(pos, end);
int sOff = partitioningScheme.getOffset(pos);
int eOff = partitioningScheme.getEndOffset(start, endPos);
var p = pages[page++];
for (long i = sOff; i < eOff; i++) {
consumer.accept(i + (pos - sOff), p.get(i));
}
}
}
@Override
@ -176,12 +193,46 @@ public class PagingLongArray extends AbstractPagingArray<LongArrayPage, LongBuff
@Override
public void transformEach(long start, long end, LongTransformer transformer) {
delegateToEachPage(start, end, (page, s, e) -> page.transformEach(s, e, transformer));
assert end >= start;
int page = partitioningScheme.getPage(start);
long endPos;
for (long pos = start; pos < end; pos = endPos) {
endPos = partitioningScheme.getPageEnd(pos, end);
int sOff = partitioningScheme.getOffset(pos);
int eOff = partitioningScheme.getEndOffset(start, endPos);
var p = pages[page++];
for (long i = sOff; i < eOff; i++) {
p.set(i, transformer.transform(i + (pos - sOff), p.get(i)));
}
}
}
@Override
public void transformEachIO(long start, long end, LongIOTransformer transformer) throws IOException {
delegateToEachPageIO(start, end, (page, s, e) -> page.transformEachIO(s, e, transformer));
assert end >= start;
int page = partitioningScheme.getPage(start);
long endPos;
for (long pos = start; pos < end; pos = endPos) {
endPos = partitioningScheme.getPageEnd(pos, end);
int sOff = partitioningScheme.getOffset(pos);
int eOff = partitioningScheme.getEndOffset(start, endPos);
var p = pages[page++];
for (long i = sOff; i < eOff; i++) {
p.set(i, transformer.transform(i + (pos - sOff), p.get(i)));
}
}
}
@Override

100
code/libraries/array/src/test/java/nu/marginalia/array/algo/IntArrayTransformations2Test.java Normal file
View File

@ -0,0 +1,100 @@
package nu.marginalia.array.algo;
import nu.marginalia.array.IntArray;
import nu.marginalia.array.LongArray;
import nu.marginalia.array.page.IntArrayPage;
import nu.marginalia.array.page.LongArrayPage;
import nu.marginalia.array.page.PagingIntArray;
import nu.marginalia.array.page.PagingLongArray;
import nu.marginalia.array.scheme.PowerOf2PartitioningScheme;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.ArrayList;
import static org.junit.jupiter.api.Assertions.assertEquals;
class IntArrayTransformations2Test {
IntArray basic;
IntArray paged;
IntArray shifted;
final int size = 1026;
@BeforeEach
public void setUp() {
basic = IntArrayPage.onHeap(size);
paged = PagingIntArray.newOnHeap(new PowerOf2PartitioningScheme(32), size);
shifted = IntArrayPage.onHeap(size + 30).shifted(30);
int[] vals = new int[size];
for (int i = 0; i < vals.length; i++) {
vals[i] = i+10;
}
basic.set(0, vals);
paged.set(0, vals);
shifted.set(0, vals);
}
@Test
void forEach() {
foreachTester(basic);
foreachTester(paged);
foreachTester(shifted);
}
@Test
void transformEach() {
transformTester(basic);
transformTester(paged);
transformTester(shifted);
}
@Test
void transformEachIO() throws IOException {
transformTesterIO(basic);
transformTesterIO(paged);
transformTesterIO(shifted);
}
private void transformTester(IntArray array) {
ArrayList<Long> offsets = new ArrayList<>();
array.transformEach(0, size, (i, val) -> {
assertEquals(i+10, val);
offsets.add(i);
return -val;
});
for (int i = 0; i < size; i++) {
assertEquals(-(i+10), array.get(i));
}
}
private void transformTesterIO(IntArray array) throws IOException {
ArrayList<Long> offsets = new ArrayList<>();
array.transformEachIO(0, size, (i, val) -> {
assertEquals(i+10, val);
offsets.add(i);
return -val;
});
for (int i = 0; i < size; i++) {
assertEquals(-(i+10), array.get(i));
}
for (int i = 0; i < size; i++) {
assertEquals(offsets.get(i), i);
}
}
private void foreachTester(IntArray array) {
ArrayList<Long> offsets = new ArrayList<>();
array.forEach(0, size, (i, val) -> {
assertEquals(i+10, val);
offsets.add(i);
});
for (int i = 0; i < size; i++) {
assertEquals(offsets.get(i), i);
}
}
}

32
code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortNTest.java
View File

@ -27,9 +27,7 @@ class LongArraySortNTest {
LongArray paged;
LongArray shifted;
Long2ObjectOpenHashMap<LongOpenHashSet> basicPairs;
Long2ObjectOpenHashMap<LongOpenHashSet> pagedPairs;
Long2ObjectOpenHashMap<LongOpenHashSet> shiftedPairs;
Long2ObjectOpenHashMap<LongOpenHashSet> dataAsPairs;
final int size = 1026;
@ -48,13 +46,11 @@ class LongArraySortNTest {
values[i] = -values[i];
}
basic.transformEach(0, size, (i, old) -> values[(int) i]);
paged.transformEach(0, size, (i, old) -> values[(int) i]);
shifted.transformEach(0, size, (i, old) -> values[(int) i]);
basic.set(0, values);
paged.set(0, values);
shifted.set(0, values);
basicPairs = asPairs(basic);
pagedPairs = asPairs(paged);
shiftedPairs = asPairs(shifted);
dataAsPairs = asPairs(basic);
}
@ -144,9 +140,9 @@ class LongArraySortNTest {
shifted.insertionSortN(2, 0, size);
assertTrue(shifted.isSortedN(2, 0, size));
compare(basic, basicPairs);
compare(paged, pagedPairs);
compare(shifted, shiftedPairs);
compare(basic, dataAsPairs);
compare(paged, dataAsPairs);
compare(shifted, dataAsPairs);
}
@Test
@ -160,9 +156,9 @@ class LongArraySortNTest {
shifted.quickSortN(2, 0, size);
assertTrue(shifted.isSortedN(2, 0, size));
compare(basic, basicPairs);
compare(paged, pagedPairs);
compare(shifted, shiftedPairs);
compare(basic, dataAsPairs);
compare(paged, dataAsPairs);
compare(shifted, dataAsPairs);
}
@Test
@ -177,9 +173,9 @@ class LongArraySortNTest {
shifted.mergeSortN(2, 0, size, Path.of("/tmp"));
assertTrue(shifted.isSortedN(2, 0, size));
compare(basic, basicPairs);
compare(paged, pagedPairs);
compare(shifted, shiftedPairs);
compare(basic, dataAsPairs);
compare(paged, dataAsPairs);
compare(shifted, dataAsPairs);
}
private Long2ObjectOpenHashMap<LongOpenHashSet> asPairs(LongArray array) {

36
code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArraySortTest.java
View File

@ -1,12 +1,12 @@
package nu.marginalia.array.algo;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import nu.marginalia.array.LongArray;
import nu.marginalia.array.page.LongArrayPage;
import nu.marginalia.array.page.PagingLongArray;
import nu.marginalia.array.scheme.PowerOf2PartitioningScheme;
import nu.marginalia.util.test.TestUtil;
import org.apache.commons.lang3.ArrayUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@ -25,6 +25,9 @@ class LongArraySortTest {
LongArray basic;
LongArray paged;
LongArray shifted;
LongOpenHashSet valueSet;
final int size = 1026;
@BeforeEach
@ -33,12 +36,19 @@ class LongArraySortTest {
paged = PagingLongArray.newOnHeap(new PowerOf2PartitioningScheme(32), size);
shifted = LongArrayPage.onHeap(size + 30).shifted(30);
valueSet = new LongOpenHashSet();
var random = new Random();
long[] values = new long[size];
for (int i = 0; i < size; i++) {
values[i] = random.nextInt(0, 1000);
valueSet.add(values[i]);
}
basic.set(0, values);
paged.set(0, values);
shifted.set(0, values);
basic.transformEach(0, size, (i, old) -> values[(int) i]);
paged.transformEach(0, size, (i, old) -> values[(int) i]);
shifted.transformEach(0, size, (i, old) -> values[(int) i]);
@ -114,6 +124,16 @@ class LongArraySortTest {
}
void verifyValuesPresent(LongArray array) {
LongOpenHashSet actualValues = new LongOpenHashSet();
for (int i = 0; i < array.size(); i++) {
actualValues.add(array.get(i));
}
assertEquals(valueSet, actualValues);
}
@Test
void insertionSort() {
basic.insertionSort(0, size);
@ -124,6 +144,10 @@ class LongArraySortTest {
shifted.insertionSort(0, size);
assertTrue(shifted.isSorted(0, 128));
verifyValuesPresent(basic);
verifyValuesPresent(paged);
verifyValuesPresent(shifted);
}
@Test
@ -136,6 +160,12 @@ class LongArraySortTest {
shifted.quickSort(0, size);
assertTrue(shifted.isSorted(0, size));
verifyValuesPresent(basic);
verifyValuesPresent(paged);
verifyValuesPresent(shifted);
}
@Test
@ -148,6 +178,10 @@ class LongArraySortTest {
shifted.mergeSort(0, size, Path.of("/tmp"));
assertTrue(shifted.isSorted(0, size));
verifyValuesPresent(basic);
verifyValuesPresent(paged);
verifyValuesPresent(shifted);
}
@Test

97
code/libraries/array/src/test/java/nu/marginalia/array/algo/LongArrayTransformations2Test.java Normal file
View File

@ -0,0 +1,97 @@
package nu.marginalia.array.algo;
import nu.marginalia.array.LongArray;
import nu.marginalia.array.page.LongArrayPage;
import nu.marginalia.array.page.PagingLongArray;
import nu.marginalia.array.scheme.PowerOf2PartitioningScheme;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.ArrayList;
import static org.junit.jupiter.api.Assertions.assertEquals;
class LongArrayTransformations2Test {
LongArray basic;
LongArray paged;
LongArray shifted;
final int size = 1026;
@BeforeEach
public void setUp() {
basic = LongArrayPage.onHeap(size);
paged = PagingLongArray.newOnHeap(new PowerOf2PartitioningScheme(32), size);
shifted = LongArrayPage.onHeap(size + 30).shifted(30);
long[] vals = new long[size];
for (int i = 0; i < vals.length; i++) {
vals[i] = i+10;
}
basic.set(0, vals);
paged.set(0, vals);
shifted.set(0, vals);
}
@Test
void forEach() {
foreachTester(basic);
foreachTester(paged);
foreachTester(shifted);
}
@Test
void transformEach() {
transformTester(basic);
transformTester(paged);
transformTester(shifted);
}
@Test
void transformEachIO() throws IOException {
transformTesterIO(basic);
transformTesterIO(paged);
transformTesterIO(shifted);
}
private void transformTester(LongArray array) {
ArrayList<Long> offsets = new ArrayList<>();
array.transformEach(0, size, (i, val) -> {
assertEquals(i+10, val);
offsets.add(i);
return -val;
});
for (int i = 0; i < size; i++) {
assertEquals(-(i+10), array.get(i));
}
}
private void transformTesterIO(LongArray array) throws IOException {
ArrayList<Long> offsets = new ArrayList<>();
array.transformEachIO(0, size, (i, val) -> {
assertEquals(i+10, val);
offsets.add(i);
return -val;
});
for (int i = 0; i < size; i++) {
assertEquals(-(i+10), array.get(i));
}
for (int i = 0; i < size; i++) {
assertEquals(offsets.get(i), i);
}
}
private void foreachTester(LongArray array) {
ArrayList<Long> offsets = new ArrayList<>();
array.forEach(0, size, (i, val) -> {
assertEquals(i+10, val);
offsets.add(i);
});
for (int i = 0; i < size; i++) {
assertEquals(offsets.get(i), i);
}
}
}