161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

(array) Remove unused 'madvise' code and 3rd party dependency on 'uppend'

Browse source 
This wasn't actually hooked in anywhere.  Removing the dependency and code.  If it turns out we need madvise in the future, we'll re-introducde it.
This commit is contained in:
Viktor Lofgren 2024-01-22 12:56:45 +01:00
parent b91ea1d7ca
commit 6a1bfd6270
13 changed files with 2 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
code/features-index/index-forward/build.gradle
View file

@ -19,8 +19,6 @@ dependencies {
implementation project(':code:common:model')
implementation project(':code:common:process')
implementation project(':third-party:uppend')
implementation libs.bundles.slf4j
implementation libs.prometheus

8
code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java
View file

@ -1,6 +1,5 @@
package nu.marginalia.index.forward;
import com.upserve.uppend.blobs.NativeIO;
import gnu.trove.map.hash.TLongIntHashMap;
import nu.marginalia.array.LongArray;
import nu.marginalia.array.LongArrayFactory;
@ -65,12 +64,7 @@ public class ForwardIndexReader {
}
private static LongArray loadData(Path dataFile) throws IOException {
var data = LongArrayFactory.mmapForReadingShared(dataFile);
// Total data is small, try to keep it in RAM for speed
data.advice(NativeIO.Advice.WillNeed);
return data;
return LongArrayFactory.mmapForReadingShared(dataFile);
}
public long getDocMeta(long docId) {

2
code/libraries/array/build.gradle
View file

@ -10,8 +10,6 @@ java {
}
dependencies {
implementation project(':third-party:uppend')
implementation libs.bundles.slf4j
implementation libs.notnull

5
code/libraries/array/src/main/java/nu/marginalia/array/IntArray.java
View file

@ -1,6 +1,5 @@
package nu.marginalia.array;
import com.upserve.uppend.blobs.NativeIO;
import nu.marginalia.array.algo.IntArrayBase;
import nu.marginalia.array.algo.IntArraySearch;
import nu.marginalia.array.algo.IntArraySort;
@ -30,9 +29,5 @@ public interface IntArray extends IntArrayBase, IntArrayTransformations, IntArra
void force();
void advice(NativeIO.Advice advice) throws IOException;
void advice(NativeIO.Advice advice, long start, long end) throws IOException;
default void close() { }
}

5
code/libraries/array/src/main/java/nu/marginalia/array/LongArray.java
View file

@ -1,6 +1,5 @@
package nu.marginalia.array;
import com.upserve.uppend.blobs.NativeIO;
import nu.marginalia.array.algo.LongArrayBase;
import nu.marginalia.array.algo.LongArraySearch;
import nu.marginalia.array.algo.LongArraySort;
@ -8,7 +7,6 @@ import nu.marginalia.array.algo.LongArrayTransformations;
import nu.marginalia.array.delegate.ShiftedLongArray;
import nu.marginalia.array.page.SegmentLongArray;
import java.io.IOException;
import java.lang.foreign.Arena;
@ -32,7 +30,4 @@ public interface LongArray extends LongArrayBase, LongArrayTransformations, Long
void force();
void close();
void advice(NativeIO.Advice advice) throws IOException;
void advice(NativeIO.Advice advice, long start, long end) throws IOException;
}

11
code/libraries/array/src/main/java/nu/marginalia/array/delegate/ReferenceImplIntArrayDelegate.java
View file

@ -1,6 +1,5 @@
package nu.marginalia.array.delegate;
import com.upserve.uppend.blobs.NativeIO;
import nu.marginalia.array.ArrayRangeReference;
import nu.marginalia.array.IntArray;
@ -51,14 +50,4 @@ public class ReferenceImplIntArrayDelegate implements IntArray {
delegate.force();
}
@Override
public void advice(NativeIO.Advice advice) throws IOException {
delegate.advice(advice);
}
@Override
public void advice(NativeIO.Advice advice, long start, long end) throws IOException {
delegate.advice(advice, start, end);
}
}

12
code/libraries/array/src/main/java/nu/marginalia/array/delegate/ReferenceImplLongArrayDelegate.java
View file

@ -1,6 +1,5 @@
package nu.marginalia.array.delegate;
import com.upserve.uppend.blobs.NativeIO;
import nu.marginalia.array.ArrayRangeReference;
import nu.marginalia.array.LongArray;
@ -51,17 +50,6 @@ public class ReferenceImplLongArrayDelegate implements LongArray {
delegate.force();
}
@Override
public void advice(NativeIO.Advice advice) throws IOException {
delegate.advice(advice);
}
@Override
public void advice(NativeIO.Advice advice, long start, long end) throws IOException {
delegate.advice(advice, start, end);
}
@Override
public void close() {
delegate.close();

11
code/libraries/array/src/main/java/nu/marginalia/array/delegate/ShiftedIntArray.java
View file

@ -1,6 +1,5 @@
package nu.marginalia.array.delegate;
import com.upserve.uppend.blobs.NativeIO;
import nu.marginalia.array.ArrayRangeReference;
import nu.marginalia.array.IntArray;
import nu.marginalia.array.algo.SortingContext;
@ -210,14 +209,4 @@ public class ShiftedIntArray implements IntArray {
delegate.force();
}
@Override
public void advice(NativeIO.Advice advice) throws IOException {
delegate.advice(advice, shift, shift + size());
}
@Override
public void advice(NativeIO.Advice advice, long start, long end) throws IOException {
delegate.advice(advice, start + shift, end + shift);
}
}

11
code/libraries/array/src/main/java/nu/marginalia/array/delegate/ShiftedLongArray.java
View file

@ -1,6 +1,5 @@
package nu.marginalia.array.delegate;
import com.upserve.uppend.blobs.NativeIO;
import nu.marginalia.array.ArrayRangeReference;
import nu.marginalia.array.LongArray;
import nu.marginalia.array.algo.LongArraySearch;
@ -282,14 +281,4 @@ public class ShiftedLongArray implements LongArray {
delegate.force();
}
@Override
public void advice(NativeIO.Advice advice) throws IOException {
delegate.advice(advice, shift, shift + size());
}
@Override
public void advice(NativeIO.Advice advice, long start, long end) throws IOException {
delegate.advice(advice, start + shift, end + shift);
}
}

13
code/libraries/array/src/main/java/nu/marginalia/array/page/SegmentIntArray.java
View file

@ -1,9 +1,7 @@
package nu.marginalia.array.page;
import com.upserve.uppend.blobs.NativeIO;
import nu.marginalia.array.ArrayRangeReference;
import nu.marginalia.array.IntArray;
import nu.marginalia.array.IntArray;
import javax.annotation.Nullable;
import java.io.IOException;
@ -11,7 +9,6 @@ import java.lang.foreign.Arena;
import java.lang.foreign.MemorySegment;
import java.nio.ByteBuffer;
import java.nio.IntBuffer;
import java.nio.LongBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.OpenOption;
@ -176,14 +173,4 @@ public class SegmentIntArray implements PartitionPage, IntArray {
}
@Override
public void advice(NativeIO.Advice advice) throws IOException {
// NativeIO.madvise((MappedByteBuffer) byteBuffer, advice);
}
@Override
public void advice(NativeIO.Advice advice, long start, long end) throws IOException {
// NativeIO.madviseRange((MappedByteBuffer) byteBuffer, advice, (int) start, (int) (end-start));
}
}

11
code/libraries/array/src/main/java/nu/marginalia/array/page/SegmentLongArray.java
View file

@ -1,6 +1,5 @@
package nu.marginalia.array.page;
import com.upserve.uppend.blobs.NativeIO;
import nu.marginalia.array.ArrayRangeReference;
import nu.marginalia.array.LongArray;
@ -174,14 +173,4 @@ public class SegmentLongArray implements PartitionPage, LongArray {
}
@Override
public void advice(NativeIO.Advice advice) throws IOException {
// NativeIO.madvise((MappedByteBuffer) byteBuffer, advice);
}
@Override
public void advice(NativeIO.Advice advice, long start, long end) throws IOException {
// NativeIO.madviseRange((MappedByteBuffer) byteBuffer, advice, (int) start, (int) (end-start));
}
}

1
settings.gradle
View file

@ -92,7 +92,6 @@ include 'third-party:porterstemmer'
include 'third-party:xz'
include 'third-party:symspell'
include 'third-party:rdrpostagger'
include 'third-party:uppend'
include 'third-party:openzim'
include 'third-party:count-min-sketch'
include 'third-party:monkey-patch-opennlp'

2
third-party/README.md vendored
View file

@ -8,10 +8,10 @@ or lack an artifact, or to override some default that is inappropriate for the t
### Modified
* [RDRPosTagger](rdrpostagger/) - GPL3
* [PorterStemmer](porterstemmer/) - LGPL3
* [Uppend](uppend/) - MIT
* [OpenZIM](openzim/) - GPL-2.0+
* [Commons Codec](commons-codec/) - Apache 2.0
* [encylopedia.marginalia.nu](encyclopedia-marginalia-nu/) - GPL 2.0+
### Repackaged
* [SymSpell](symspell/) - LGPL-3.0
* [Count-Min-Sketch](count-min-sketch/) - Apache 2.0