161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

More tests for BTree, cleaned up code a bit.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-03-05 13:03:55 +01:00
parent 96f6cd19e9
commit 4d94a023c9
8 changed files with 233 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
libraries/btree/build.gradle
View File

@ -16,6 +16,7 @@ dependencies {
annotationProcessor libs.lombok
implementation libs.bundles.slf4j
testImplementation project(':libraries:misc')
testImplementation libs.bundles.slf4j.test
testImplementation libs.bundles.junit
testImplementation libs.mockito

52
libraries/btree/src/main/java/nu/marginalia/btree/BTreeReader.java
View File

@ -43,48 +43,35 @@ public class BTreeReader {
/** Keeps all items in buffer that exist in the btree */
public void retainEntries(LongQueryBuffer buffer) {
BTreePointer pointer = new BTreePointer(header);
if (header.layers() == 0) {
BTreePointer pointer = new BTreePointer(header);
while (buffer.hasMore()) {
pointer.retainData(buffer);
}
}
retainSingle(buffer);
}
/** Removes all items in buffer that exist in the btree */
public void rejectEntries(LongQueryBuffer buffer) {
if (header.layers() == 0) {
BTreePointer pointer = new BTreePointer(header);
while (buffer.hasMore()) {
pointer.rejectData(buffer);
}
}
rejectSingle(buffer);
}
private void retainSingle(LongQueryBuffer buffer) {
BTreePointer pointer = new BTreePointer(header);
for (; buffer.hasMore(); pointer.resetToRoot()) {
else while (buffer.hasMore()) {
long val = buffer.currentValue();
if (!pointer.walkToData(val)) {
buffer.rejectAndAdvance();
continue;
}
else {
pointer.retainData(buffer);
}
pointer.retainData(buffer);
pointer.resetToRoot();
}
}
private void rejectSingle(LongQueryBuffer buffer) {
/** Removes all items in buffer that exist in the btree */
public void rejectEntries(LongQueryBuffer buffer) {
BTreePointer pointer = new BTreePointer(header);
for (; buffer.hasMore(); pointer.resetToRoot()) {
if (header.layers() == 0) {
while (buffer.hasMore()) {
pointer.rejectData(buffer);
}
}
else while (buffer.hasMore()) {
long val = buffer.currentValue();
if (pointer.walkToData(val) && pointer.containsData(val)) {
@ -93,6 +80,8 @@ public class BTreeReader {
else {
buffer.retainAndAdvance();
}
pointer.resetToRoot();
}
}
@ -132,10 +121,10 @@ public class BTreeReader {
}
if (header.layers() == 0) {
return queryJustIndex(keys, offset);
return queryDataNoIndex(keys, offset);
}
else {
return queryBtree(keys, offset);
return queryDataWithIndex(keys, offset);
}
}
@ -147,7 +136,8 @@ public class BTreeReader {
return true;
}
private long[] queryJustIndex(long[] keys, int offset) {
// This b-tree doesn't have any index and is actually just a sorted list of items
private long[] queryDataNoIndex(long[] keys, int offset) {
long[] ret = new long[keys.length];
long searchStart = 0;
@ -164,7 +154,7 @@ public class BTreeReader {
return ret;
}
private long[] queryBtree(long[] keys, int offset) {
private long[] queryDataWithIndex(long[] keys, int offset) {
BTreePointer pointer = new BTreePointer(header);
long[] ret = new long[keys.length];

44
libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderQueryDataWithIndexTest.java Normal file
View File

@ -0,0 +1,44 @@
package nu.marginalia.btree;
import nu.marginalia.array.LongArray;
import nu.marginalia.array.buffer.LongQueryBuffer;
import nu.marginalia.btree.model.BTreeBlockSize;
import nu.marginalia.btree.model.BTreeContext;
import nu.marginalia.util.PrimeUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.Arrays;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class BTreeReaderQueryDataWithIndexTest {
BTreeContext ctx = new BTreeContext(5, 2, BTreeBlockSize.BS_32);
LongArray array;
@BeforeEach
public void setUp() throws IOException {
array = LongArray.allocate(65536);
new BTreeWriter(array, ctx).write(0, 1000, slice -> {
for (int idx = 0; idx < 1000; idx++) {
slice.set(idx * 2, 2 * idx);
slice.set(idx * 2 + 1, 5 * idx);
}
});
// we expect index[key] = 5 * key / 2;
}
@Test
public void testQueryData() {
long[] keys = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
BTreeReader reader = new BTreeReader(array, ctx, 0);
long[] data = reader.queryData(keys, 1);
assertArrayEquals(data, new long[] { 0, 5, 0, 10, 0, 15, 0, 20, 0, 25 });
}
}

40
libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderQueryDataWithoutIndexTest.java Normal file
View File

@ -0,0 +1,40 @@
package nu.marginalia.btree;
import nu.marginalia.array.LongArray;
import nu.marginalia.btree.model.BTreeBlockSize;
import nu.marginalia.btree.model.BTreeContext;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
public class BTreeReaderQueryDataWithoutIndexTest {
BTreeContext ctx = new BTreeContext(5, 2, BTreeBlockSize.BS_2048);
LongArray array;
@BeforeEach
public void setUp() throws IOException {
array = LongArray.allocate(65536);
new BTreeWriter(array, ctx).write(0, 1000, slice -> {
for (int idx = 0; idx < 1000; idx++) {
slice.set(idx * 2, 2 * idx);
slice.set(idx * 2 + 1, 5 * idx);
}
});
// we expect index[key] = 5 * key / 2;
}
@Test
public void testQueryData() {
long[] keys = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
BTreeReader reader = new BTreeReader(array, ctx, 0);
long[] data = reader.queryData(keys, 1);
assertArrayEquals(data, new long[] { 0, 5, 0, 10, 0, 15, 0, 20, 0, 25 });
}
}

59
libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderRejectRetainWithIndexTest.java Normal file
View File

@ -0,0 +1,59 @@
package nu.marginalia.btree;
import nu.marginalia.array.LongArray;
import nu.marginalia.array.buffer.LongQueryBuffer;
import nu.marginalia.btree.model.BTreeBlockSize;
import nu.marginalia.btree.model.BTreeContext;
import nu.marginalia.util.PrimeUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.Arrays;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
public class BTreeReaderRejectRetainWithIndexTest {
BTreeContext ctx = new BTreeContext(5, 1, BTreeBlockSize.BS_32);
LongArray array;
@BeforeEach
public void setUp() throws IOException {
array = LongArray.allocate(65536);
new BTreeWriter(array, ctx).write(0, 1000, slice -> {
int p = 2;
for (int idx = 0; idx < 1000; idx++) {
slice.set(idx, p);
p = (int) PrimeUtil.nextPrime(p + 1, 1);
}
});
}
@Test
public void testRetain() {
LongQueryBuffer odds = new LongQueryBuffer(50);
Arrays.setAll(odds.data, i -> 2L*i + 1);
BTreeReader reader = new BTreeReader(array, ctx, 0);
reader.retainEntries(odds);
odds.finalizeFiltering();
long[] primeOdds = odds.copyData();
long[] first100OddPrimes = new long[] { 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97 };
assertArrayEquals(first100OddPrimes, primeOdds);
}
@Test
public void testReject() {
LongQueryBuffer odds = new LongQueryBuffer(50);
Arrays.setAll(odds.data, i -> 2L*i + 1);
BTreeReader reader = new BTreeReader(array, ctx, 0);
reader.rejectEntries(odds);
odds.finalizeFiltering();
long[] nonPrimeOdds = odds.copyData();
long[] first100OddNonPrimes = new long[] { 1, 9, 15, 21, 25, 27, 33, 35, 39, 45, 49, 51, 55, 57, 63, 65, 69, 75, 77, 81, 85, 87, 91, 93, 95, 99 };
assertArrayEquals(first100OddNonPrimes, nonPrimeOdds);
}
}

59
libraries/btree/src/test/java/nu/marginalia/btree/BTreeReaderRejectRetainWithoutIndexTest.java Normal file
View File

@ -0,0 +1,59 @@
package nu.marginalia.btree;
import nu.marginalia.array.LongArray;
import nu.marginalia.array.buffer.LongQueryBuffer;
import nu.marginalia.btree.model.BTreeBlockSize;
import nu.marginalia.btree.model.BTreeContext;
import nu.marginalia.util.PrimeUtil;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.Arrays;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
public class BTreeReaderRejectRetainWithoutIndexTest {
BTreeContext ctx = new BTreeContext(5, 1, BTreeBlockSize.BS_2048);
LongArray array;
@BeforeEach
public void setUp() throws IOException {
array = LongArray.allocate(65536);
new BTreeWriter(array, ctx).write(0, 1000, slice -> {
int p = 2;
for (int idx = 0; idx < 1000; idx++) {
slice.set(idx, p);
p = (int) PrimeUtil.nextPrime(p + 1, 1);
}
});
}
@Test
public void testRetain() {
LongQueryBuffer odds = new LongQueryBuffer(50);
Arrays.setAll(odds.data, i -> 2L*i + 1);
BTreeReader reader = new BTreeReader(array, ctx, 0);
reader.retainEntries(odds);
odds.finalizeFiltering();
long[] primeOdds = odds.copyData();
long[] first100OddPrimes = new long[] { 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97 };
assertArrayEquals(first100OddPrimes, primeOdds);
}
@Test
public void testReject() {
LongQueryBuffer odds = new LongQueryBuffer(50);
Arrays.setAll(odds.data, i -> 2L*i + 1);
BTreeReader reader = new BTreeReader(array, ctx, 0);
reader.rejectEntries(odds);
odds.finalizeFiltering();
long[] nonPrimeOdds = odds.copyData();
long[] first100OddNonPrimes = new long[] { 1, 9, 15, 21, 25, 27, 33, 35, 39, 45, 49, 51, 55, 57, 63, 65, 69, 75, 77, 81, 85, 87, 91, 93, 95, 99 };
assertArrayEquals(first100OddNonPrimes, nonPrimeOdds);
}
}

4
libraries/btree/src/test/java/nu/marginalia/btree/BTreeWriterTest.java
View File

@ -63,7 +63,7 @@ class BTreeWriterTest {
var header = writer.makeHeader(0, i);
printTreeLayout(i, header, ctx);
// printTreeLayout(i, header, ctx);
if (header.layers() >= 1) {
assertEquals(1, ctx.indexLayerSize(i, header.layers() - 1) / ctx.pageSize());
@ -145,7 +145,7 @@ class BTreeWriterTest {
var reader = new BTreeReader(array, ctx, 0);
printTreeLayout(data.length, reader.getHeader(), ctx);
// printTreeLayout(data.length, reader.getHeader(), ctx);
for (int i = 0; i < data.length; i++) {
long offset = reader.findEntry(data[i]);

12
libraries/misc/src/main/java/nu/marginalia/util/PrimeUtil.java
View File

@ -3,13 +3,15 @@ package nu.marginalia.util;
// This is not a fast way of finding primes
public class PrimeUtil {
public static long nextPrime(long start, long step) {
if (isDivisible(start, 2)) {
start = start + step;
/** Returns the next prime value starting at start. If start is prime, return start.
*/
public static long nextPrime(long start, long direction) {
if (isCoprime(start, 2)) {
start = start + direction;
}
long val;
for (val = start; !isPrime(val); val += 2*step) {}
for (val = start; !isPrime(val); val += 2*direction) {}
return val;
}
@ -28,7 +30,7 @@ public class PrimeUtil {
return true;
}
public static boolean isDivisible(long a, long b) {
public static boolean isCoprime(long a, long b) {
if (a == 0 || b == 0) {
return false;
}