From 73eaa0865d742979de47a83ca7671333254d7944 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Sun, 12 Mar 2023 10:50:31 +0100 Subject: [PATCH] The refactoring will continue until morale improves. --- LICENSE.md | 1 - code/api/index-api/build.gradle | 2 +- .../common/build.gradle | 2 +- code/{crawl => crawl-models}/common/readme.md | 0 .../crawling/common/AbortMonitor.java | 0 .../marginalia/crawling/common/TaskStats.java | 0 .../marginalia/crawling/common/WorkLog.java | 0 .../common/blocklist/GeoIpBlocklist.java | 0 .../common/blocklist/InetAddressCache.java | 0 .../common/blocklist/IpBlockList.java | 0 .../common/blocklist/UrlBlocklist.java | 0 .../crawling/common/link/LinkParser.java | 0 .../crawling/common/model/HtmlStandard.java | 0 .../crawling/common/plan/CrawlPlanLoader.java | 0 .../plan/CrawlerSpecificationLoader.java | 0 .../crawling/common/plan/EdgeCrawlPlan.java | 0 .../src/main/resources/log4j2.properties | 0 .../converting-model/build.gradle | 2 +- code/crawl-models/converting-model/readme.md | 4 +++ .../converting/instruction/Instruction.java | 0 .../instruction/InstructionTag.java | 0 .../converting/instruction/Interpreter.java | 0 .../instruction/instructions/DomainLink.java | 0 .../instruction/instructions/LoadDomain.java | 0 .../instructions/LoadDomainLink.java | 0 .../instructions/LoadDomainRedirect.java | 0 .../instructions/LoadKeywords.java | 0 .../instructions/LoadProcessedDocument.java | 0 .../LoadProcessedDocumentWithError.java | 0 .../instructions/LoadProcessedDomain.java | 0 .../instruction/instructions/LoadRssFeed.java | 0 .../instruction/instructions/LoadUrl.java | 0 .../converting/model/DocumentKeywords.java | 0 .../model/DocumentKeywordsBuilder.java | 0 .../crawling-model/build.gradle | 0 .../crawling-model/readme.md | 4 +-- .../crawling/io/CrawledDomainReader.java | 0 .../crawling/io/CrawledDomainWriter.java | 0 .../crawling/model/ContentType.java | 0 .../crawling/model/CrawlLogEntry.java | 0 .../crawling/model/CrawledDocument.java | 0 .../crawling/model/CrawledDomain.java | 0 .../crawling/model/CrawlerDocumentStatus.java | 0 .../crawling/model/CrawlerDomainStatus.java | 0 .../crawling/model/CrawlingSpecification.java | 0 .../crawling/model/SerializableCrawlData.java | 0 code/crawl/converting-model/readme.md | 4 --- code/crawl/converting-process/build.gradle | 12 +++---- .../crawl-job-extractor-process/build.gradle | 4 +-- code/crawl/crawling-process/build.gradle | 6 ++-- code/crawl/experimental/build.gradle | 10 +++--- code/crawl/loading-process/build.gradle | 10 +++--- code/crawl/readme.md | 4 +-- .../adblock/build.gradle | 0 .../adblock/readme.md | 0 .../marginalia/adblock/AdblockSimulator.java | 0 .../adblock/GoogleAnwersSpamDetector.java | 0 .../pubdate/build.gradle | 2 +- .../pubdate/readme.md | 0 .../pubdate/PubDateEffortLevel.java | 0 .../marginalia/pubdate/PubDateHeuristic.java | 0 .../nu/marginalia/pubdate/PubDateParser.java | 0 .../nu/marginalia/pubdate/PubDateSniffer.java | 0 .../PubDateHeuristicDOMParsingPass1.java | 0 .../PubDateHeuristicDOMParsingPass2.java | 0 ...PubDateHeuristicGuessFromHtmlStandard.java | 0 .../PubDateHeuristicHtml5AnyTimeTag.java | 0 .../PubDateHeuristicHtml5ArticleDateTag.java | 0 .../PubDateHeuristicHtml5ItempropDateTag.java | 0 .../heuristic/PubDateHeuristicJSONLD.java | 0 .../PubDateHeuristicLastModified.java | 0 .../heuristic/PubDateHeuristicMicrodata.java | 0 .../heuristic/PubDateHeuristicOpenGraph.java | 0 .../heuristic/PubDateHeuristicRDFaTag.java | 0 .../PubDateHeuristicUrlPatternPass1.java | 0 .../PubDateHeuristicUrlPatternPass2.java | 0 .../pubdate/PubDateSnifferTest.java | 0 code/features-crawl/readme.md | 10 ++++++ .../topic-detection/build.gradle | 0 .../topic-detection/readme.md | 0 .../nu/marginalia/topic/RecipeDetector.java | 0 .../topic/TextileCraftDetector.java | 0 .../marginalia/topic/WoodworkingDetector.java | 0 .../domain-ranking/build.gradle | 0 .../domain-ranking/readme.md | 0 .../nu/marginalia/ranking/DomainRankings.java | 0 .../marginalia/ranking/RankingAlgorithm.java | 0 .../marginalia/ranking/ReversePageRank.java | 0 .../marginalia/ranking/StandardPageRank.java | 0 .../accumulator/RankingResultAccumulator.java | 0 .../RankingResultBitSetAccumulator.java | 0 .../RankingResultHashMapAccumulator.java | 0 .../RankingResultHashSetAccumulator.java | 0 .../RankingResultListAccumulator.java | 0 .../ranking/data/RankingDomainData.java | 0 .../ranking/data/RankingDomainFetcher.java | 0 ...RankingDomainFetcherForSimilarityData.java | 0 .../tool/CreateBrowseDomainRanksTool.java | 0 .../ranking/tool/PerusePageRankV2.java | 0 .../ranking/tool/PrintDomainRanksTool.java | 0 .../ranking/tool/UpdateDomainRanksTool.java | 0 .../index-forward/build.gradle | 8 ++--- .../index-forward/readme.md | 0 .../index/forward/ForwardIndexConverter.java | 0 .../index/forward/ForwardIndexParameters.java | 0 .../index/forward/ForwardIndexReader.java | 0 .../forward/ParamMatchingQueryFilter.java | 0 .../forward/ForwardIndexConverterTest.java | 0 .../java/nu/marginalia/test/TestUtil.java | 0 .../index-journal/build.gradle | 2 +- .../index-journal/readme.md | 0 .../journal/model/IndexJournalEntry.java | 0 .../model/IndexJournalEntryBuilder.java | 0 .../journal/model/IndexJournalEntryData.java | 0 .../model/IndexJournalEntryHeader.java | 0 .../journal/model/IndexJournalFileHeader.java | 0 .../journal/model/IndexJournalStatistics.java | 0 .../journal/reader/IndexJournalReadEntry.java | 0 .../journal/reader/IndexJournalReader.java | 0 ...ndexJournalReaderSingleCompressedFile.java | 0 .../journal/writer/IndexJournalWriter.java | 0 .../writer/IndexJournalWriterImpl.java | 0 .../index/journal/IndexJournalTest.java | 0 .../index-query/build.gradle | 0 .../index-query/readme.md | 0 .../index/query/EmptyEntrySource.java | 0 .../marginalia/index/query/EntrySource.java | 0 .../nu/marginalia/index/query/IndexQuery.java | 0 .../index/query/IndexQueryBuilder.java | 0 .../index/query/IndexQueryParams.java | 0 .../index/query/IndexSearchBudget.java | 0 .../index/query/filter/QueryFilterAnyOf.java | 0 .../query/filter/QueryFilterLetThrough.java | 0 .../index/query/filter/QueryFilterNoPass.java | 0 .../QueryFilterStepExcludeFromPredicate.java | 0 .../filter/QueryFilterStepFromPredicate.java | 0 .../index/query/filter/QueryFilterStepIf.java | 0 .../index/query/limit/QueryLimits.java | 0 .../index/query/limit/QueryStrategy.java | 0 .../index/query/limit/SpecificationLimit.java | 0 .../query/limit/SpecificationLimitType.java | 0 .../marginalia/index/searchset/SearchSet.java | 0 .../index-reverse/build.gradle | 8 ++--- .../index-reverse/readme.md | 0 .../index/reverse/ReverseIndexConverter.java | 0 .../index/reverse/ReverseIndexParameters.java | 0 .../index/reverse/ReverseIndexPrioReader.java | 0 .../ReverseIndexPriorityParameters.java | 0 .../index/reverse/ReverseIndexReader.java | 0 .../query/ReverseIndexEntrySource.java | 0 .../ReverseIndexEntrySourceBehavior.java | 0 .../query/ReverseIndexRejectFilter.java | 0 .../query/ReverseIndexRetainFilter.java | 0 .../reverse/ReverseIndexConverterTest.java | 0 .../reverse/ReverseIndexConverterTest2.java | 0 .../java/nu/marginalia/test/TestUtil.java | 0 .../lexicon/build.gradle | 0 .../lexicon/readme.md | 0 .../nu/marginalia/dict/DictionaryData.java | 0 .../marginalia/dict/DictionaryDataBank.java | 0 .../nu/marginalia/dict/DictionaryMap.java | 0 .../dict/OffHeapDictionaryHashMap.java | 0 .../marginalia/dict/OnHeapDictionaryMap.java | 0 .../nu/marginalia/lexicon/KeywordLexicon.java | 0 .../lexicon/KeywordLexiconReadOnlyView.java | 0 .../journal/KeywordLexiconJournal.java | 0 .../KeywordLexiconJournalCommitQueue.java | 0 .../journal/KeywordLexiconJournalFile.java | 0 .../lexicon/KeywordLexiconTest.java | 0 code/{index => features-index}/readme.md | 9 ++++- .../query-parser/build.gradle | 0 .../query-parser/readme.md | 0 .../marginalia/query_parser/QueryParser.java | 0 .../query_parser/QueryPermutation.java | 0 .../query_parser/QueryTokenizer.java | 0 .../query_parser/QueryVariants.java | 0 .../marginalia/query_parser/token/Token.java | 0 .../query_parser/token/TokenType.java | 0 .../transform_list/TransformList.java | 0 .../query_parser/BodyQueryParserTest.java | 0 .../query_parser/QueryParserTest.java | 0 .../query_parser/QueryVariantsTest.java | 0 .../transform_list/TransformListTest.java | 0 .../marginalia/util/TestLanguageModels.java | 0 .../random-websites/build.gradle | 0 .../browse/DbBrowseDomainsFromUrlId.java | 0 .../browse/DbBrowseDomainsRandom.java | 0 .../browse/DbBrowseDomainsSimilarCosine.java | 0 .../browse/DbBrowseDomainsSimilarOldAlgo.java | 0 .../browse/experimental/AndCardIntSet.java | 0 .../EdgeDomainLinkConsineSimilarityMain.java | 0 .../EdgeWordWordConsineSimilarityMain.java | 0 .../marginalia/browse/model/BrowseResult.java | 0 .../browse/model/BrowseResultSet.java | 0 .../experimental/AndCardIntSetTest.java | 0 code/{features => features-search}/readme.md | 9 ++--- .../screenshots/build.gradle | 0 .../screenshot/ScreenshotService.java | 0 code/libraries/LICENSE.txt | 15 ++++++++ code/libraries/readme.md | 7 +++- code/readme.md | 25 ++++++++++--- .../assistant-service/build.gradle | 2 +- code/services-core/index-service/build.gradle | 13 ++++--- code/services-core/index-service/readme.md | 8 +++-- .../services-core/search-service/build.gradle | 8 ++--- .../dating-service/build.gradle | 4 +-- .../dating-service/readme.md | 4 +-- .../explorer-service/build.gradle | 2 +- .../explorer-service/readme.md | 4 +-- settings.gradle | 35 ++++++++++--------- tools/screenshot/build.gradle | 2 +- 211 files changed, 148 insertions(+), 94 deletions(-) rename code/{crawl => crawl-models}/common/build.gradle (94%) rename code/{crawl => crawl-models}/common/readme.md (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/AbortMonitor.java (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/TaskStats.java (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/WorkLog.java (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/blocklist/GeoIpBlocklist.java (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/blocklist/InetAddressCache.java (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/blocklist/IpBlockList.java (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/blocklist/UrlBlocklist.java (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/link/LinkParser.java (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/model/HtmlStandard.java (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/plan/CrawlPlanLoader.java (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/plan/CrawlerSpecificationLoader.java (100%) rename code/{crawl => crawl-models}/common/src/main/java/nu/marginalia/crawling/common/plan/EdgeCrawlPlan.java (100%) rename code/{crawl => crawl-models}/common/src/main/resources/log4j2.properties (100%) rename code/{crawl => crawl-models}/converting-model/build.gradle (94%) create mode 100644 code/crawl-models/converting-model/readme.md rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/Instruction.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/InstructionTag.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/Interpreter.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/DomainLink.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomain.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomainLink.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomainRedirect.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadKeywords.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDocument.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDocumentWithError.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDomain.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadRssFeed.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadUrl.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/model/DocumentKeywords.java (100%) rename code/{crawl => crawl-models}/converting-model/src/main/java/nu/marginalia/converting/model/DocumentKeywordsBuilder.java (100%) rename code/{crawl => crawl-models}/crawling-model/build.gradle (100%) rename code/{crawl => crawl-models}/crawling-model/readme.md (74%) rename code/{crawl => crawl-models}/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainReader.java (100%) rename code/{crawl => crawl-models}/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainWriter.java (100%) rename code/{crawl => crawl-models}/crawling-model/src/main/java/nu/marginalia/crawling/model/ContentType.java (100%) rename code/{crawl => crawl-models}/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlLogEntry.java (100%) rename code/{crawl => crawl-models}/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDocument.java (100%) rename code/{crawl => crawl-models}/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDomain.java (100%) rename code/{crawl => crawl-models}/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDocumentStatus.java (100%) rename code/{crawl => crawl-models}/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDomainStatus.java (100%) rename code/{crawl => crawl-models}/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlingSpecification.java (100%) rename code/{crawl => crawl-models}/crawling-model/src/main/java/nu/marginalia/crawling/model/SerializableCrawlData.java (100%) delete mode 100644 code/crawl/converting-model/readme.md rename code/{features => features-crawl}/adblock/build.gradle (100%) rename code/{features => features-crawl}/adblock/readme.md (100%) rename code/{features => features-crawl}/adblock/src/main/java/nu/marginalia/adblock/AdblockSimulator.java (100%) rename code/{features => features-crawl}/adblock/src/main/java/nu/marginalia/adblock/GoogleAnwersSpamDetector.java (100%) rename code/{features => features-crawl}/pubdate/build.gradle (93%) rename code/{features => features-crawl}/pubdate/readme.md (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/PubDateEffortLevel.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/PubDateHeuristic.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/PubDateParser.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/PubDateSniffer.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicJSONLD.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicLastModified.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicMicrodata.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicOpenGraph.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicRDFaTag.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java (100%) rename code/{features => features-crawl}/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java (100%) rename code/{features => features-crawl}/pubdate/src/test/java/nu/marginalia/pubdate/PubDateSnifferTest.java (100%) create mode 100644 code/features-crawl/readme.md rename code/{features => features-crawl}/topic-detection/build.gradle (100%) rename code/{features => features-crawl}/topic-detection/readme.md (100%) rename code/{features => features-crawl}/topic-detection/src/main/java/nu/marginalia/topic/RecipeDetector.java (100%) rename code/{features => features-crawl}/topic-detection/src/main/java/nu/marginalia/topic/TextileCraftDetector.java (100%) rename code/{features => features-crawl}/topic-detection/src/main/java/nu/marginalia/topic/WoodworkingDetector.java (100%) rename code/{features => features-index}/domain-ranking/build.gradle (100%) rename code/{features => features-index}/domain-ranking/readme.md (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/ReversePageRank.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/StandardPageRank.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainData.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainFetcher.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainFetcherForSimilarityData.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/tool/CreateBrowseDomainRanksTool.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/tool/PerusePageRankV2.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/tool/PrintDomainRanksTool.java (100%) rename code/{features => features-index}/domain-ranking/src/main/java/nu/marginalia/ranking/tool/UpdateDomainRanksTool.java (100%) rename code/{index => features-index}/index-forward/build.gradle (77%) rename code/{index => features-index}/index-forward/readme.md (100%) rename code/{index => features-index}/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java (100%) rename code/{index => features-index}/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java (100%) rename code/{index => features-index}/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java (100%) rename code/{index => features-index}/index-forward/src/main/java/nu/marginalia/index/forward/ParamMatchingQueryFilter.java (100%) rename code/{index => features-index}/index-forward/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java (100%) rename code/{index => features-index}/index-forward/src/test/java/nu/marginalia/test/TestUtil.java (100%) rename code/{index => features-index}/index-journal/build.gradle (93%) rename code/{index => features-index}/index-journal/readme.md (100%) rename code/{index => features-index}/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntry.java (100%) rename code/{index => features-index}/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryBuilder.java (100%) rename code/{index => features-index}/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryData.java (100%) rename code/{index => features-index}/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryHeader.java (100%) rename code/{index => features-index}/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalFileHeader.java (100%) rename code/{index => features-index}/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalStatistics.java (100%) rename code/{index => features-index}/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReadEntry.java (100%) rename code/{index => features-index}/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReader.java (100%) rename code/{index => features-index}/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReaderSingleCompressedFile.java (100%) rename code/{index => features-index}/index-journal/src/main/java/nu.marginalia.index/journal/writer/IndexJournalWriter.java (100%) rename code/{index => features-index}/index-journal/src/main/java/nu.marginalia.index/journal/writer/IndexJournalWriterImpl.java (100%) rename code/{index => features-index}/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java (100%) rename code/{index => features-index}/index-query/build.gradle (100%) rename code/{index => features-index}/index-query/readme.md (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/EntrySource.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/IndexQuery.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/IndexQueryParams.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java (100%) rename code/{index => features-index}/index-query/src/main/java/nu/marginalia/index/searchset/SearchSet.java (100%) rename code/{index => features-index}/index-reverse/build.gradle (76%) rename code/{index => features-index}/index-reverse/readme.md (100%) rename code/{index => features-index}/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexConverter.java (100%) rename code/{index => features-index}/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexParameters.java (100%) rename code/{index => features-index}/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexPrioReader.java (100%) rename code/{index => features-index}/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexPriorityParameters.java (100%) rename code/{index => features-index}/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexReader.java (100%) rename code/{index => features-index}/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexEntrySource.java (100%) rename code/{index => features-index}/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexEntrySourceBehavior.java (100%) rename code/{index => features-index}/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexRejectFilter.java (100%) rename code/{index => features-index}/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexRetainFilter.java (100%) rename code/{index => features-index}/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexConverterTest.java (100%) rename code/{index => features-index}/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexConverterTest2.java (100%) rename code/{index => features-index}/index-reverse/src/test/java/nu/marginalia/test/TestUtil.java (100%) rename code/{index => features-index}/lexicon/build.gradle (100%) rename code/{index => features-index}/lexicon/readme.md (100%) rename code/{index => features-index}/lexicon/src/main/java/nu/marginalia/dict/DictionaryData.java (100%) rename code/{index => features-index}/lexicon/src/main/java/nu/marginalia/dict/DictionaryDataBank.java (100%) rename code/{index => features-index}/lexicon/src/main/java/nu/marginalia/dict/DictionaryMap.java (100%) rename code/{index => features-index}/lexicon/src/main/java/nu/marginalia/dict/OffHeapDictionaryHashMap.java (100%) rename code/{index => features-index}/lexicon/src/main/java/nu/marginalia/dict/OnHeapDictionaryMap.java (100%) rename code/{index => features-index}/lexicon/src/main/java/nu/marginalia/lexicon/KeywordLexicon.java (100%) rename code/{index => features-index}/lexicon/src/main/java/nu/marginalia/lexicon/KeywordLexiconReadOnlyView.java (100%) rename code/{index => features-index}/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournal.java (100%) rename code/{index => features-index}/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournalCommitQueue.java (100%) rename code/{index => features-index}/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournalFile.java (100%) rename code/{index => features-index}/lexicon/src/test/java/nu/marginalia/lexicon/KeywordLexiconTest.java (100%) rename code/{index => features-index}/readme.md (85%) rename code/{features => features-search}/query-parser/build.gradle (100%) rename code/{features => features-search}/query-parser/readme.md (100%) rename code/{features => features-search}/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java (100%) rename code/{features => features-search}/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java (100%) rename code/{features => features-search}/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java (100%) rename code/{features => features-search}/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java (100%) rename code/{features => features-search}/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java (100%) rename code/{features => features-search}/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java (100%) rename code/{features => features-search}/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java (100%) rename code/{features => features-search}/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java (100%) rename code/{features => features-search}/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java (100%) rename code/{features => features-search}/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java (100%) rename code/{features => features-search}/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java (100%) rename code/{features => features-search}/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java (100%) rename code/{features => features-search}/random-websites/build.gradle (100%) rename code/{features => features-search}/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsFromUrlId.java (100%) rename code/{features => features-search}/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java (100%) rename code/{features => features-search}/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java (100%) rename code/{features => features-search}/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarOldAlgo.java (100%) rename code/{features => features-search}/random-websites/src/main/java/nu/marginalia/browse/experimental/AndCardIntSet.java (100%) rename code/{features => features-search}/random-websites/src/main/java/nu/marginalia/browse/experimental/EdgeDomainLinkConsineSimilarityMain.java (100%) rename code/{features => features-search}/random-websites/src/main/java/nu/marginalia/browse/experimental/EdgeWordWordConsineSimilarityMain.java (100%) rename code/{features => features-search}/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResult.java (100%) rename code/{features => features-search}/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResultSet.java (100%) rename code/{features => features-search}/random-websites/src/test/java/nu/marginalia/experimental/AndCardIntSetTest.java (100%) rename code/{features => features-search}/readme.md (73%) rename code/{features => features-search}/screenshots/build.gradle (100%) rename code/{features => features-search}/screenshots/src/main/java/nu/marginalia/screenshot/ScreenshotService.java (100%) create mode 100644 code/libraries/LICENSE.txt diff --git a/LICENSE.md b/LICENSE.md index bdf0773d..dd844411 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -13,4 +13,3 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - diff --git a/code/api/index-api/build.gradle b/code/api/index-api/build.gradle index 73520b36..c495cef8 100644 --- a/code/api/index-api/build.gradle +++ b/code/api/index-api/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':code:common:service-discovery') implementation project(':code:common:service-client') - implementation project(':code:index:index-query') + implementation project(':code:features-index:index-query') implementation libs.lombok annotationProcessor libs.lombok diff --git a/code/crawl/common/build.gradle b/code/crawl-models/common/build.gradle similarity index 94% rename from code/crawl/common/build.gradle rename to code/crawl-models/common/build.gradle index 483dbee8..da9a2eab 100644 --- a/code/crawl/common/build.gradle +++ b/code/crawl-models/common/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:config') implementation project(':code:libraries:guarded-regex') - implementation project(':code:crawl:crawling-model') + implementation project(':code:crawl-models:crawling-model') implementation libs.notnull implementation libs.lombok diff --git a/code/crawl/common/readme.md b/code/crawl-models/common/readme.md similarity index 100% rename from code/crawl/common/readme.md rename to code/crawl-models/common/readme.md diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/AbortMonitor.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/AbortMonitor.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/AbortMonitor.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/AbortMonitor.java diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/TaskStats.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/TaskStats.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/TaskStats.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/TaskStats.java diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/WorkLog.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/WorkLog.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/WorkLog.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/WorkLog.java diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/blocklist/GeoIpBlocklist.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/blocklist/GeoIpBlocklist.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/blocklist/GeoIpBlocklist.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/blocklist/GeoIpBlocklist.java diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/blocklist/InetAddressCache.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/blocklist/InetAddressCache.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/blocklist/InetAddressCache.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/blocklist/InetAddressCache.java diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/blocklist/IpBlockList.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/blocklist/IpBlockList.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/blocklist/IpBlockList.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/blocklist/IpBlockList.java diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/blocklist/UrlBlocklist.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/blocklist/UrlBlocklist.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/blocklist/UrlBlocklist.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/blocklist/UrlBlocklist.java diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/link/LinkParser.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/link/LinkParser.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/link/LinkParser.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/link/LinkParser.java diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/model/HtmlStandard.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/model/HtmlStandard.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/model/HtmlStandard.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/model/HtmlStandard.java diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/plan/CrawlPlanLoader.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/plan/CrawlPlanLoader.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/plan/CrawlPlanLoader.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/plan/CrawlPlanLoader.java diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/plan/CrawlerSpecificationLoader.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/plan/CrawlerSpecificationLoader.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/plan/CrawlerSpecificationLoader.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/plan/CrawlerSpecificationLoader.java diff --git a/code/crawl/common/src/main/java/nu/marginalia/crawling/common/plan/EdgeCrawlPlan.java b/code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/plan/EdgeCrawlPlan.java similarity index 100% rename from code/crawl/common/src/main/java/nu/marginalia/crawling/common/plan/EdgeCrawlPlan.java rename to code/crawl-models/common/src/main/java/nu/marginalia/crawling/common/plan/EdgeCrawlPlan.java diff --git a/code/crawl/common/src/main/resources/log4j2.properties b/code/crawl-models/common/src/main/resources/log4j2.properties similarity index 100% rename from code/crawl/common/src/main/resources/log4j2.properties rename to code/crawl-models/common/src/main/resources/log4j2.properties diff --git a/code/crawl/converting-model/build.gradle b/code/crawl-models/converting-model/build.gradle similarity index 94% rename from code/crawl/converting-model/build.gradle rename to code/crawl-models/converting-model/build.gradle index 29bbd025..00b604c3 100644 --- a/code/crawl/converting-model/build.gradle +++ b/code/crawl-models/converting-model/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':code:common:service-client') implementation project(':code:libraries:language-processing') - implementation project(':code:crawl:common') + implementation project(':code:crawl-models:common') implementation libs.lombok annotationProcessor libs.lombok diff --git a/code/crawl-models/converting-model/readme.md b/code/crawl-models/converting-model/readme.md new file mode 100644 index 00000000..a38b68d3 --- /dev/null +++ b/code/crawl-models/converting-model/readme.md @@ -0,0 +1,4 @@ +# Converting Models + +Contains models shared by the [converting-process](../../crawl/converting-process/) and +[loading-process](../../crawl/loading-process/). \ No newline at end of file diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/Instruction.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/Instruction.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/Instruction.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/Instruction.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/InstructionTag.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/InstructionTag.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/InstructionTag.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/InstructionTag.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/Interpreter.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/Interpreter.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/Interpreter.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/Interpreter.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/DomainLink.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/DomainLink.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/DomainLink.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/DomainLink.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomain.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomain.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomain.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomain.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomainLink.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomainLink.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomainLink.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomainLink.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomainRedirect.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomainRedirect.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomainRedirect.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadDomainRedirect.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadKeywords.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadKeywords.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadKeywords.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadKeywords.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDocument.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDocument.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDocument.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDocument.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDocumentWithError.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDocumentWithError.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDocumentWithError.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDocumentWithError.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDomain.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDomain.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDomain.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadProcessedDomain.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadRssFeed.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadRssFeed.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadRssFeed.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadRssFeed.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadUrl.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadUrl.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadUrl.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/instruction/instructions/LoadUrl.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/model/DocumentKeywords.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/model/DocumentKeywords.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/model/DocumentKeywords.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/model/DocumentKeywords.java diff --git a/code/crawl/converting-model/src/main/java/nu/marginalia/converting/model/DocumentKeywordsBuilder.java b/code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/model/DocumentKeywordsBuilder.java similarity index 100% rename from code/crawl/converting-model/src/main/java/nu/marginalia/converting/model/DocumentKeywordsBuilder.java rename to code/crawl-models/converting-model/src/main/java/nu/marginalia/converting/model/DocumentKeywordsBuilder.java diff --git a/code/crawl/crawling-model/build.gradle b/code/crawl-models/crawling-model/build.gradle similarity index 100% rename from code/crawl/crawling-model/build.gradle rename to code/crawl-models/crawling-model/build.gradle diff --git a/code/crawl/crawling-model/readme.md b/code/crawl-models/crawling-model/readme.md similarity index 74% rename from code/crawl/crawling-model/readme.md rename to code/crawl-models/crawling-model/readme.md index 601db53e..663b612a 100644 --- a/code/crawl/crawling-model/readme.md +++ b/code/crawl-models/crawling-model/readme.md @@ -1,7 +1,7 @@ # Crawling Models -Contains models shared by the [crawling-process](../crawling-process/) and -[converting-process](../converting-process/). +Contains models shared by the [crawling-process](../../crawl/crawling-process/) and +[converting-process](../../crawl/converting-process/). ## Central Classes diff --git a/code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainReader.java b/code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainReader.java similarity index 100% rename from code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainReader.java rename to code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainReader.java diff --git a/code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainWriter.java b/code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainWriter.java similarity index 100% rename from code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainWriter.java rename to code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/io/CrawledDomainWriter.java diff --git a/code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/ContentType.java b/code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/ContentType.java similarity index 100% rename from code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/ContentType.java rename to code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/ContentType.java diff --git a/code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlLogEntry.java b/code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlLogEntry.java similarity index 100% rename from code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlLogEntry.java rename to code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlLogEntry.java diff --git a/code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDocument.java b/code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDocument.java similarity index 100% rename from code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDocument.java rename to code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDocument.java diff --git a/code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDomain.java b/code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDomain.java similarity index 100% rename from code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDomain.java rename to code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawledDomain.java diff --git a/code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDocumentStatus.java b/code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDocumentStatus.java similarity index 100% rename from code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDocumentStatus.java rename to code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDocumentStatus.java diff --git a/code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDomainStatus.java b/code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDomainStatus.java similarity index 100% rename from code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDomainStatus.java rename to code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlerDomainStatus.java diff --git a/code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlingSpecification.java b/code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlingSpecification.java similarity index 100% rename from code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlingSpecification.java rename to code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/CrawlingSpecification.java diff --git a/code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/SerializableCrawlData.java b/code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/SerializableCrawlData.java similarity index 100% rename from code/crawl/crawling-model/src/main/java/nu/marginalia/crawling/model/SerializableCrawlData.java rename to code/crawl-models/crawling-model/src/main/java/nu/marginalia/crawling/model/SerializableCrawlData.java diff --git a/code/crawl/converting-model/readme.md b/code/crawl/converting-model/readme.md deleted file mode 100644 index f52a3b42..00000000 --- a/code/crawl/converting-model/readme.md +++ /dev/null @@ -1,4 +0,0 @@ -# Converting Models - -Contains models shared by the [converting-process](../converting-process/) and -[loading-process](../loading-process/). \ No newline at end of file diff --git a/code/crawl/converting-process/build.gradle b/code/crawl/converting-process/build.gradle index e33f0b44..261e3a4e 100644 --- a/code/crawl/converting-process/build.gradle +++ b/code/crawl/converting-process/build.gradle @@ -33,13 +33,13 @@ dependencies { implementation project(':code:libraries:big-string') implementation project(':code:libraries:language-processing') - implementation project(':code:crawl:common') - implementation project(':code:crawl:converting-model') - implementation project(':code:crawl:crawling-model') + implementation project(':code:crawl-models:common') + implementation project(':code:crawl-models:converting-model') + implementation project(':code:crawl-models:crawling-model') - implementation project(':code:features:adblock') - implementation project(':code:features:pubdate') - implementation project(':code:features:topic-detection') + implementation project(':code:features-crawl:adblock') + implementation project(':code:features-crawl:pubdate') + implementation project(':code:features-crawl:topic-detection') implementation libs.lombok annotationProcessor libs.lombok diff --git a/code/crawl/crawl-job-extractor-process/build.gradle b/code/crawl/crawl-job-extractor-process/build.gradle index 166430e9..85e67459 100644 --- a/code/crawl/crawl-job-extractor-process/build.gradle +++ b/code/crawl/crawl-job-extractor-process/build.gradle @@ -21,8 +21,8 @@ tasks.distZip.enabled = false dependencies { implementation project(':code:common:model') implementation project(':code:common:service') - implementation project(':code:crawl:crawling-model') - implementation project(':code:crawl:common') + implementation project(':code:crawl-models:crawling-model') + implementation project(':code:crawl-models:common') implementation libs.lombok annotationProcessor libs.lombok diff --git a/code/crawl/crawling-process/build.gradle b/code/crawl/crawling-process/build.gradle index 29630437..c4207316 100644 --- a/code/crawl/crawling-process/build.gradle +++ b/code/crawl/crawling-process/build.gradle @@ -27,9 +27,9 @@ dependencies { implementation project(':code:common:service-discovery') implementation project(':code:common:service-client') implementation project(':code:libraries:language-processing') - implementation project(':code:crawl:common') - implementation project(':code:crawl:crawling-model') - implementation project(':code:crawl:converting-model') + implementation project(':code:crawl-models:common') + implementation project(':code:crawl-models:crawling-model') + implementation project(':code:crawl-models:converting-model') implementation libs.lombok annotationProcessor libs.lombok diff --git a/code/crawl/experimental/build.gradle b/code/crawl/experimental/build.gradle index 85063928..8a7c5c1f 100644 --- a/code/crawl/experimental/build.gradle +++ b/code/crawl/experimental/build.gradle @@ -20,11 +20,13 @@ dependencies { implementation project(':code:common:service-discovery') implementation project(':code:common:service-client') implementation project(':code:libraries:language-processing') - implementation project(':code:crawl:common') - implementation project(':code:crawl:crawling-model') + + implementation project(':code:crawl-models:common') + implementation project(':code:crawl-models:crawling-model') + implementation project(':code:crawl:converting-process') - implementation project(':code:features:adblock') - implementation project(':code:features:topic-detection') + implementation project(':code:features-crawl:adblock') + implementation project(':code:features-crawl:topic-detection') implementation libs.lombok annotationProcessor libs.lombok diff --git a/code/crawl/loading-process/build.gradle b/code/crawl/loading-process/build.gradle index ea233dda..60894727 100644 --- a/code/crawl/loading-process/build.gradle +++ b/code/crawl/loading-process/build.gradle @@ -24,15 +24,15 @@ dependencies { implementation project(':code:common:service') implementation project(':code:common:service-discovery') implementation project(':code:common:service-client') - implementation project(':code:index:lexicon') - implementation project(':code:index:index-journal') + implementation project(':code:features-index:lexicon') + implementation project(':code:features-index:index-journal') implementation project(':code:libraries:language-processing') testImplementation project(':code:services-core:search-service') - implementation project(':code:crawl:common') - implementation project(':code:crawl:crawling-model') - implementation project(':code:crawl:converting-model') + implementation project(':code:crawl-models:common') + implementation project(':code:crawl-models:crawling-model') + implementation project(':code:crawl-models:converting-model') implementation libs.lombok diff --git a/code/crawl/readme.md b/code/crawl/readme.md index c5d3016e..254c817b 100644 --- a/code/crawl/readme.md +++ b/code/crawl/readme.md @@ -8,13 +8,13 @@ based on the content in the database. ## 2. Crawl Process The [crawling-process](crawling-process/) fetches website contents and saves them -as compressed JSON models described in [crawling-model](crawling-model/). +as compressed JSON models described in [crawling-model](../crawl-models/crawling-model/). ## 3. Converting Process The [converting-process](converting-process/) reads crawl data from the crawling step and processes them, extracting keywords and metadata and saves them as compressed JSON models -described in [converting-model](converting-model/). +described in [converting-model](../crawl-models/converting-model/). ## 4. Loading Process diff --git a/code/features/adblock/build.gradle b/code/features-crawl/adblock/build.gradle similarity index 100% rename from code/features/adblock/build.gradle rename to code/features-crawl/adblock/build.gradle diff --git a/code/features/adblock/readme.md b/code/features-crawl/adblock/readme.md similarity index 100% rename from code/features/adblock/readme.md rename to code/features-crawl/adblock/readme.md diff --git a/code/features/adblock/src/main/java/nu/marginalia/adblock/AdblockSimulator.java b/code/features-crawl/adblock/src/main/java/nu/marginalia/adblock/AdblockSimulator.java similarity index 100% rename from code/features/adblock/src/main/java/nu/marginalia/adblock/AdblockSimulator.java rename to code/features-crawl/adblock/src/main/java/nu/marginalia/adblock/AdblockSimulator.java diff --git a/code/features/adblock/src/main/java/nu/marginalia/adblock/GoogleAnwersSpamDetector.java b/code/features-crawl/adblock/src/main/java/nu/marginalia/adblock/GoogleAnwersSpamDetector.java similarity index 100% rename from code/features/adblock/src/main/java/nu/marginalia/adblock/GoogleAnwersSpamDetector.java rename to code/features-crawl/adblock/src/main/java/nu/marginalia/adblock/GoogleAnwersSpamDetector.java diff --git a/code/features/pubdate/build.gradle b/code/features-crawl/pubdate/build.gradle similarity index 93% rename from code/features/pubdate/build.gradle rename to code/features-crawl/pubdate/build.gradle index caac0252..57b21cf6 100644 --- a/code/features/pubdate/build.gradle +++ b/code/features-crawl/pubdate/build.gradle @@ -15,7 +15,7 @@ java { dependencies { implementation project(':code:common:model') - implementation project(':code:crawl:common') + implementation project(':code:crawl-models:common') implementation libs.lombok annotationProcessor libs.lombok diff --git a/code/features/pubdate/readme.md b/code/features-crawl/pubdate/readme.md similarity index 100% rename from code/features/pubdate/readme.md rename to code/features-crawl/pubdate/readme.md diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/PubDateEffortLevel.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/PubDateEffortLevel.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/PubDateEffortLevel.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/PubDateEffortLevel.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/PubDateHeuristic.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/PubDateHeuristic.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/PubDateHeuristic.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/PubDateHeuristic.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/PubDateParser.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/PubDateParser.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/PubDateParser.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/PubDateParser.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/PubDateSniffer.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/PubDateSniffer.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/PubDateSniffer.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/PubDateSniffer.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass1.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicDOMParsingPass2.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicGuessFromHtmlStandard.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5AnyTimeTag.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ArticleDateTag.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicHtml5ItempropDateTag.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicJSONLD.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicJSONLD.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicJSONLD.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicJSONLD.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicLastModified.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicLastModified.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicLastModified.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicLastModified.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicMicrodata.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicMicrodata.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicMicrodata.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicMicrodata.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicOpenGraph.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicOpenGraph.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicOpenGraph.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicOpenGraph.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicRDFaTag.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicRDFaTag.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicRDFaTag.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicRDFaTag.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass1.java diff --git a/code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java b/code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java similarity index 100% rename from code/features/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java rename to code/features-crawl/pubdate/src/main/java/nu/marginalia/pubdate/heuristic/PubDateHeuristicUrlPatternPass2.java diff --git a/code/features/pubdate/src/test/java/nu/marginalia/pubdate/PubDateSnifferTest.java b/code/features-crawl/pubdate/src/test/java/nu/marginalia/pubdate/PubDateSnifferTest.java similarity index 100% rename from code/features/pubdate/src/test/java/nu/marginalia/pubdate/PubDateSnifferTest.java rename to code/features-crawl/pubdate/src/test/java/nu/marginalia/pubdate/PubDateSnifferTest.java diff --git a/code/features-crawl/readme.md b/code/features-crawl/readme.md new file mode 100644 index 00000000..bb93a75a --- /dev/null +++ b/code/features-crawl/readme.md @@ -0,0 +1,10 @@ +# Crawl Features + +These are bits of search-engine related code that are relatively isolated pieces of business logic, +that benefit from the clarity of being kept separate from the rest of the crawling code. + + +* [adblock](adblock/) - Simulates Adblock +* [pubdate](pubdate/) - Determines when a document was published +* [topic-detection](topic-detection/) - Tries to identify the topic of a website + diff --git a/code/features/topic-detection/build.gradle b/code/features-crawl/topic-detection/build.gradle similarity index 100% rename from code/features/topic-detection/build.gradle rename to code/features-crawl/topic-detection/build.gradle diff --git a/code/features/topic-detection/readme.md b/code/features-crawl/topic-detection/readme.md similarity index 100% rename from code/features/topic-detection/readme.md rename to code/features-crawl/topic-detection/readme.md diff --git a/code/features/topic-detection/src/main/java/nu/marginalia/topic/RecipeDetector.java b/code/features-crawl/topic-detection/src/main/java/nu/marginalia/topic/RecipeDetector.java similarity index 100% rename from code/features/topic-detection/src/main/java/nu/marginalia/topic/RecipeDetector.java rename to code/features-crawl/topic-detection/src/main/java/nu/marginalia/topic/RecipeDetector.java diff --git a/code/features/topic-detection/src/main/java/nu/marginalia/topic/TextileCraftDetector.java b/code/features-crawl/topic-detection/src/main/java/nu/marginalia/topic/TextileCraftDetector.java similarity index 100% rename from code/features/topic-detection/src/main/java/nu/marginalia/topic/TextileCraftDetector.java rename to code/features-crawl/topic-detection/src/main/java/nu/marginalia/topic/TextileCraftDetector.java diff --git a/code/features/topic-detection/src/main/java/nu/marginalia/topic/WoodworkingDetector.java b/code/features-crawl/topic-detection/src/main/java/nu/marginalia/topic/WoodworkingDetector.java similarity index 100% rename from code/features/topic-detection/src/main/java/nu/marginalia/topic/WoodworkingDetector.java rename to code/features-crawl/topic-detection/src/main/java/nu/marginalia/topic/WoodworkingDetector.java diff --git a/code/features/domain-ranking/build.gradle b/code/features-index/domain-ranking/build.gradle similarity index 100% rename from code/features/domain-ranking/build.gradle rename to code/features-index/domain-ranking/build.gradle diff --git a/code/features/domain-ranking/readme.md b/code/features-index/domain-ranking/readme.md similarity index 100% rename from code/features/domain-ranking/readme.md rename to code/features-index/domain-ranking/readme.md diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/DomainRankings.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/RankingAlgorithm.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/ReversePageRank.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/ReversePageRank.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/ReversePageRank.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/ReversePageRank.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/StandardPageRank.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/StandardPageRank.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/StandardPageRank.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/StandardPageRank.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultAccumulator.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultBitSetAccumulator.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashMapAccumulator.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultHashSetAccumulator.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/accumulator/RankingResultListAccumulator.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainData.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainData.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainData.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainData.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainFetcher.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainFetcher.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainFetcher.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainFetcher.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainFetcherForSimilarityData.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainFetcherForSimilarityData.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainFetcherForSimilarityData.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/data/RankingDomainFetcherForSimilarityData.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/tool/CreateBrowseDomainRanksTool.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/tool/CreateBrowseDomainRanksTool.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/tool/CreateBrowseDomainRanksTool.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/tool/CreateBrowseDomainRanksTool.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/tool/PerusePageRankV2.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/tool/PerusePageRankV2.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/tool/PerusePageRankV2.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/tool/PerusePageRankV2.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/tool/PrintDomainRanksTool.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/tool/PrintDomainRanksTool.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/tool/PrintDomainRanksTool.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/tool/PrintDomainRanksTool.java diff --git a/code/features/domain-ranking/src/main/java/nu/marginalia/ranking/tool/UpdateDomainRanksTool.java b/code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/tool/UpdateDomainRanksTool.java similarity index 100% rename from code/features/domain-ranking/src/main/java/nu/marginalia/ranking/tool/UpdateDomainRanksTool.java rename to code/features-index/domain-ranking/src/main/java/nu/marginalia/ranking/tool/UpdateDomainRanksTool.java diff --git a/code/index/index-forward/build.gradle b/code/features-index/index-forward/build.gradle similarity index 77% rename from code/index/index-forward/build.gradle rename to code/features-index/index-forward/build.gradle index 011a39ac..e7a34566 100644 --- a/code/index/index-forward/build.gradle +++ b/code/features-index/index-forward/build.gradle @@ -13,10 +13,10 @@ java { dependencies { implementation project(':code:libraries:array') implementation project(':code:libraries:btree') - implementation project(':code:features:domain-ranking') - implementation project(':code:index:index-query') - implementation project(':code:index:index-journal') - implementation project(':code:index:lexicon') + implementation project(':code:features-index:domain-ranking') + implementation project(':code:features-index:index-query') + implementation project(':code:features-index:index-journal') + implementation project(':code:features-index:lexicon') implementation project(':code:common:model') implementation project(':third-party:uppend') diff --git a/code/index/index-forward/readme.md b/code/features-index/index-forward/readme.md similarity index 100% rename from code/index/index-forward/readme.md rename to code/features-index/index-forward/readme.md diff --git a/code/index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java b/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java similarity index 100% rename from code/index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java rename to code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexConverter.java diff --git a/code/index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java b/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java similarity index 100% rename from code/index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java rename to code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexParameters.java diff --git a/code/index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java b/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java similarity index 100% rename from code/index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java rename to code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ForwardIndexReader.java diff --git a/code/index/index-forward/src/main/java/nu/marginalia/index/forward/ParamMatchingQueryFilter.java b/code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ParamMatchingQueryFilter.java similarity index 100% rename from code/index/index-forward/src/main/java/nu/marginalia/index/forward/ParamMatchingQueryFilter.java rename to code/features-index/index-forward/src/main/java/nu/marginalia/index/forward/ParamMatchingQueryFilter.java diff --git a/code/index/index-forward/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java b/code/features-index/index-forward/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java similarity index 100% rename from code/index/index-forward/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java rename to code/features-index/index-forward/src/test/java/nu/marginalia/index/forward/ForwardIndexConverterTest.java diff --git a/code/index/index-forward/src/test/java/nu/marginalia/test/TestUtil.java b/code/features-index/index-forward/src/test/java/nu/marginalia/test/TestUtil.java similarity index 100% rename from code/index/index-forward/src/test/java/nu/marginalia/test/TestUtil.java rename to code/features-index/index-forward/src/test/java/nu/marginalia/test/TestUtil.java diff --git a/code/index/index-journal/build.gradle b/code/features-index/index-journal/build.gradle similarity index 93% rename from code/index/index-journal/build.gradle rename to code/features-index/index-journal/build.gradle index 9aeaa209..5a7315ac 100644 --- a/code/index/index-journal/build.gradle +++ b/code/features-index/index-journal/build.gradle @@ -13,7 +13,7 @@ java { dependencies { implementation project(':code:libraries:array') implementation project(':code:common:model') - implementation project(':code:index:lexicon') + implementation project(':code:features-index:lexicon') implementation libs.lombok annotationProcessor libs.lombok diff --git a/code/index/index-journal/readme.md b/code/features-index/index-journal/readme.md similarity index 100% rename from code/index/index-journal/readme.md rename to code/features-index/index-journal/readme.md diff --git a/code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntry.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntry.java similarity index 100% rename from code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntry.java rename to code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntry.java diff --git a/code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryBuilder.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryBuilder.java similarity index 100% rename from code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryBuilder.java rename to code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryBuilder.java diff --git a/code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryData.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryData.java similarity index 100% rename from code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryData.java rename to code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryData.java diff --git a/code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryHeader.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryHeader.java similarity index 100% rename from code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryHeader.java rename to code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryHeader.java diff --git a/code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalFileHeader.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalFileHeader.java similarity index 100% rename from code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalFileHeader.java rename to code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalFileHeader.java diff --git a/code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalStatistics.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalStatistics.java similarity index 100% rename from code/index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalStatistics.java rename to code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalStatistics.java diff --git a/code/index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReadEntry.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReadEntry.java similarity index 100% rename from code/index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReadEntry.java rename to code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReadEntry.java diff --git a/code/index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReader.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReader.java similarity index 100% rename from code/index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReader.java rename to code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReader.java diff --git a/code/index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReaderSingleCompressedFile.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReaderSingleCompressedFile.java similarity index 100% rename from code/index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReaderSingleCompressedFile.java rename to code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/reader/IndexJournalReaderSingleCompressedFile.java diff --git a/code/index/index-journal/src/main/java/nu.marginalia.index/journal/writer/IndexJournalWriter.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/writer/IndexJournalWriter.java similarity index 100% rename from code/index/index-journal/src/main/java/nu.marginalia.index/journal/writer/IndexJournalWriter.java rename to code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/writer/IndexJournalWriter.java diff --git a/code/index/index-journal/src/main/java/nu.marginalia.index/journal/writer/IndexJournalWriterImpl.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/writer/IndexJournalWriterImpl.java similarity index 100% rename from code/index/index-journal/src/main/java/nu.marginalia.index/journal/writer/IndexJournalWriterImpl.java rename to code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/writer/IndexJournalWriterImpl.java diff --git a/code/index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java b/code/features-index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java similarity index 100% rename from code/index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java rename to code/features-index/index-journal/src/test/java/nu/marginalia/index/journal/IndexJournalTest.java diff --git a/code/index/index-query/build.gradle b/code/features-index/index-query/build.gradle similarity index 100% rename from code/index/index-query/build.gradle rename to code/features-index/index-query/build.gradle diff --git a/code/index/index-query/readme.md b/code/features-index/index-query/readme.md similarity index 100% rename from code/index/index-query/readme.md rename to code/features-index/index-query/readme.md diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/EmptyEntrySource.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/EntrySource.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/EntrySource.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/EntrySource.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/EntrySource.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/IndexQuery.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQuery.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/IndexQuery.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQuery.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryBuilder.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryParams.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryParams.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryParams.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexQueryParams.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/IndexSearchBudget.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterAnyOf.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterLetThrough.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterNoPass.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepExcludeFromPredicate.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepFromPredicate.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/filter/QueryFilterStepIf.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryLimits.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/QueryStrategy.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimit.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/query/limit/SpecificationLimitType.java diff --git a/code/index/index-query/src/main/java/nu/marginalia/index/searchset/SearchSet.java b/code/features-index/index-query/src/main/java/nu/marginalia/index/searchset/SearchSet.java similarity index 100% rename from code/index/index-query/src/main/java/nu/marginalia/index/searchset/SearchSet.java rename to code/features-index/index-query/src/main/java/nu/marginalia/index/searchset/SearchSet.java diff --git a/code/index/index-reverse/build.gradle b/code/features-index/index-reverse/build.gradle similarity index 76% rename from code/index/index-reverse/build.gradle rename to code/features-index/index-reverse/build.gradle index f971c6a3..3ef67762 100644 --- a/code/index/index-reverse/build.gradle +++ b/code/features-index/index-reverse/build.gradle @@ -15,10 +15,10 @@ dependencies { implementation project(':code:libraries:array') implementation project(':code:libraries:btree') implementation project(':code:libraries:random-write-funnel') - implementation project(':code:features:domain-ranking') - implementation project(':code:index:index-query') - implementation project(':code:index:index-journal') - implementation project(':code:index:lexicon') + implementation project(':code:features-index:domain-ranking') + implementation project(':code:features-index:index-query') + implementation project(':code:features-index:index-journal') + implementation project(':code:features-index:lexicon') implementation project(':code:common:model') implementation libs.lombok diff --git a/code/index/index-reverse/readme.md b/code/features-index/index-reverse/readme.md similarity index 100% rename from code/index/index-reverse/readme.md rename to code/features-index/index-reverse/readme.md diff --git a/code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexConverter.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexConverter.java similarity index 100% rename from code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexConverter.java rename to code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexConverter.java diff --git a/code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexParameters.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexParameters.java similarity index 100% rename from code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexParameters.java rename to code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexParameters.java diff --git a/code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexPrioReader.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexPrioReader.java similarity index 100% rename from code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexPrioReader.java rename to code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexPrioReader.java diff --git a/code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexPriorityParameters.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexPriorityParameters.java similarity index 100% rename from code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexPriorityParameters.java rename to code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexPriorityParameters.java diff --git a/code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexReader.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexReader.java similarity index 100% rename from code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexReader.java rename to code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/ReverseIndexReader.java diff --git a/code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexEntrySource.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexEntrySource.java similarity index 100% rename from code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexEntrySource.java rename to code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexEntrySource.java diff --git a/code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexEntrySourceBehavior.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexEntrySourceBehavior.java similarity index 100% rename from code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexEntrySourceBehavior.java rename to code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexEntrySourceBehavior.java diff --git a/code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexRejectFilter.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexRejectFilter.java similarity index 100% rename from code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexRejectFilter.java rename to code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexRejectFilter.java diff --git a/code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexRetainFilter.java b/code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexRetainFilter.java similarity index 100% rename from code/index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexRetainFilter.java rename to code/features-index/index-reverse/src/main/java/nu/marginalia/index/reverse/query/ReverseIndexRetainFilter.java diff --git a/code/index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexConverterTest.java b/code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexConverterTest.java similarity index 100% rename from code/index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexConverterTest.java rename to code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexConverterTest.java diff --git a/code/index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexConverterTest2.java b/code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexConverterTest2.java similarity index 100% rename from code/index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexConverterTest2.java rename to code/features-index/index-reverse/src/test/java/nu/marginalia/index/reverse/ReverseIndexConverterTest2.java diff --git a/code/index/index-reverse/src/test/java/nu/marginalia/test/TestUtil.java b/code/features-index/index-reverse/src/test/java/nu/marginalia/test/TestUtil.java similarity index 100% rename from code/index/index-reverse/src/test/java/nu/marginalia/test/TestUtil.java rename to code/features-index/index-reverse/src/test/java/nu/marginalia/test/TestUtil.java diff --git a/code/index/lexicon/build.gradle b/code/features-index/lexicon/build.gradle similarity index 100% rename from code/index/lexicon/build.gradle rename to code/features-index/lexicon/build.gradle diff --git a/code/index/lexicon/readme.md b/code/features-index/lexicon/readme.md similarity index 100% rename from code/index/lexicon/readme.md rename to code/features-index/lexicon/readme.md diff --git a/code/index/lexicon/src/main/java/nu/marginalia/dict/DictionaryData.java b/code/features-index/lexicon/src/main/java/nu/marginalia/dict/DictionaryData.java similarity index 100% rename from code/index/lexicon/src/main/java/nu/marginalia/dict/DictionaryData.java rename to code/features-index/lexicon/src/main/java/nu/marginalia/dict/DictionaryData.java diff --git a/code/index/lexicon/src/main/java/nu/marginalia/dict/DictionaryDataBank.java b/code/features-index/lexicon/src/main/java/nu/marginalia/dict/DictionaryDataBank.java similarity index 100% rename from code/index/lexicon/src/main/java/nu/marginalia/dict/DictionaryDataBank.java rename to code/features-index/lexicon/src/main/java/nu/marginalia/dict/DictionaryDataBank.java diff --git a/code/index/lexicon/src/main/java/nu/marginalia/dict/DictionaryMap.java b/code/features-index/lexicon/src/main/java/nu/marginalia/dict/DictionaryMap.java similarity index 100% rename from code/index/lexicon/src/main/java/nu/marginalia/dict/DictionaryMap.java rename to code/features-index/lexicon/src/main/java/nu/marginalia/dict/DictionaryMap.java diff --git a/code/index/lexicon/src/main/java/nu/marginalia/dict/OffHeapDictionaryHashMap.java b/code/features-index/lexicon/src/main/java/nu/marginalia/dict/OffHeapDictionaryHashMap.java similarity index 100% rename from code/index/lexicon/src/main/java/nu/marginalia/dict/OffHeapDictionaryHashMap.java rename to code/features-index/lexicon/src/main/java/nu/marginalia/dict/OffHeapDictionaryHashMap.java diff --git a/code/index/lexicon/src/main/java/nu/marginalia/dict/OnHeapDictionaryMap.java b/code/features-index/lexicon/src/main/java/nu/marginalia/dict/OnHeapDictionaryMap.java similarity index 100% rename from code/index/lexicon/src/main/java/nu/marginalia/dict/OnHeapDictionaryMap.java rename to code/features-index/lexicon/src/main/java/nu/marginalia/dict/OnHeapDictionaryMap.java diff --git a/code/index/lexicon/src/main/java/nu/marginalia/lexicon/KeywordLexicon.java b/code/features-index/lexicon/src/main/java/nu/marginalia/lexicon/KeywordLexicon.java similarity index 100% rename from code/index/lexicon/src/main/java/nu/marginalia/lexicon/KeywordLexicon.java rename to code/features-index/lexicon/src/main/java/nu/marginalia/lexicon/KeywordLexicon.java diff --git a/code/index/lexicon/src/main/java/nu/marginalia/lexicon/KeywordLexiconReadOnlyView.java b/code/features-index/lexicon/src/main/java/nu/marginalia/lexicon/KeywordLexiconReadOnlyView.java similarity index 100% rename from code/index/lexicon/src/main/java/nu/marginalia/lexicon/KeywordLexiconReadOnlyView.java rename to code/features-index/lexicon/src/main/java/nu/marginalia/lexicon/KeywordLexiconReadOnlyView.java diff --git a/code/index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournal.java b/code/features-index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournal.java similarity index 100% rename from code/index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournal.java rename to code/features-index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournal.java diff --git a/code/index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournalCommitQueue.java b/code/features-index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournalCommitQueue.java similarity index 100% rename from code/index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournalCommitQueue.java rename to code/features-index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournalCommitQueue.java diff --git a/code/index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournalFile.java b/code/features-index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournalFile.java similarity index 100% rename from code/index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournalFile.java rename to code/features-index/lexicon/src/main/java/nu/marginalia/lexicon/journal/KeywordLexiconJournalFile.java diff --git a/code/index/lexicon/src/test/java/nu/marginalia/lexicon/KeywordLexiconTest.java b/code/features-index/lexicon/src/test/java/nu/marginalia/lexicon/KeywordLexiconTest.java similarity index 100% rename from code/index/lexicon/src/test/java/nu/marginalia/lexicon/KeywordLexiconTest.java rename to code/features-index/lexicon/src/test/java/nu/marginalia/lexicon/KeywordLexiconTest.java diff --git a/code/index/readme.md b/code/features-index/readme.md similarity index 85% rename from code/index/readme.md rename to code/features-index/readme.md index 5953c503..19d0a3af 100644 --- a/code/index/readme.md +++ b/code/features-index/readme.md @@ -11,7 +11,14 @@ about each word, such as its position. * [index-reverse](index-reverse/) is the `word->document` index. These indices rely heavily on the [libraries/btree](../libraries/btree) and [libraries/array](../libraries/array) components. + +## Algorithms + +* [domain-ranking](domain-ranking/) contains ranking algorithms. + # Libraries + * [index-query](index-query/) contains structures for evaluating search queries. * [index-journal](index-journal/) contains tools for writing and reading index data. -* [lexicon](lexicon/) contains a mapping between words' string representation and an unique integer identifier. \ No newline at end of file +* [lexicon](lexicon/) contains a mapping between words' string representation and an unique integer identifier. + diff --git a/code/features/query-parser/build.gradle b/code/features-search/query-parser/build.gradle similarity index 100% rename from code/features/query-parser/build.gradle rename to code/features-search/query-parser/build.gradle diff --git a/code/features/query-parser/readme.md b/code/features-search/query-parser/readme.md similarity index 100% rename from code/features/query-parser/readme.md rename to code/features-search/query-parser/readme.md diff --git a/code/features/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java b/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java similarity index 100% rename from code/features/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java rename to code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryParser.java diff --git a/code/features/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java b/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java similarity index 100% rename from code/features/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java rename to code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryPermutation.java diff --git a/code/features/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java b/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java similarity index 100% rename from code/features/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java rename to code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryTokenizer.java diff --git a/code/features/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java b/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java similarity index 100% rename from code/features/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java rename to code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java diff --git a/code/features/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java b/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java similarity index 100% rename from code/features/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java rename to code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/token/Token.java diff --git a/code/features/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java b/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java similarity index 100% rename from code/features/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java rename to code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/token/TokenType.java diff --git a/code/features/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java b/code/features-search/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java similarity index 100% rename from code/features/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java rename to code/features-search/query-parser/src/main/java/nu/marginalia/transform_list/TransformList.java diff --git a/code/features/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java b/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java similarity index 100% rename from code/features/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java rename to code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java diff --git a/code/features/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java b/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java similarity index 100% rename from code/features/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java rename to code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryParserTest.java diff --git a/code/features/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java b/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java similarity index 100% rename from code/features/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java rename to code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java diff --git a/code/features/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java b/code/features-search/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java similarity index 100% rename from code/features/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java rename to code/features-search/query-parser/src/test/java/nu/marginalia/transform_list/TransformListTest.java diff --git a/code/features/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java b/code/features-search/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java similarity index 100% rename from code/features/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java rename to code/features-search/query-parser/src/test/java/nu/marginalia/util/TestLanguageModels.java diff --git a/code/features/random-websites/build.gradle b/code/features-search/random-websites/build.gradle similarity index 100% rename from code/features/random-websites/build.gradle rename to code/features-search/random-websites/build.gradle diff --git a/code/features/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsFromUrlId.java b/code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsFromUrlId.java similarity index 100% rename from code/features/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsFromUrlId.java rename to code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsFromUrlId.java diff --git a/code/features/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java b/code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java similarity index 100% rename from code/features/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java rename to code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsRandom.java diff --git a/code/features/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java b/code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java similarity index 100% rename from code/features/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java rename to code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarCosine.java diff --git a/code/features/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarOldAlgo.java b/code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarOldAlgo.java similarity index 100% rename from code/features/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarOldAlgo.java rename to code/features-search/random-websites/src/main/java/nu/marginalia/browse/DbBrowseDomainsSimilarOldAlgo.java diff --git a/code/features/random-websites/src/main/java/nu/marginalia/browse/experimental/AndCardIntSet.java b/code/features-search/random-websites/src/main/java/nu/marginalia/browse/experimental/AndCardIntSet.java similarity index 100% rename from code/features/random-websites/src/main/java/nu/marginalia/browse/experimental/AndCardIntSet.java rename to code/features-search/random-websites/src/main/java/nu/marginalia/browse/experimental/AndCardIntSet.java diff --git a/code/features/random-websites/src/main/java/nu/marginalia/browse/experimental/EdgeDomainLinkConsineSimilarityMain.java b/code/features-search/random-websites/src/main/java/nu/marginalia/browse/experimental/EdgeDomainLinkConsineSimilarityMain.java similarity index 100% rename from code/features/random-websites/src/main/java/nu/marginalia/browse/experimental/EdgeDomainLinkConsineSimilarityMain.java rename to code/features-search/random-websites/src/main/java/nu/marginalia/browse/experimental/EdgeDomainLinkConsineSimilarityMain.java diff --git a/code/features/random-websites/src/main/java/nu/marginalia/browse/experimental/EdgeWordWordConsineSimilarityMain.java b/code/features-search/random-websites/src/main/java/nu/marginalia/browse/experimental/EdgeWordWordConsineSimilarityMain.java similarity index 100% rename from code/features/random-websites/src/main/java/nu/marginalia/browse/experimental/EdgeWordWordConsineSimilarityMain.java rename to code/features-search/random-websites/src/main/java/nu/marginalia/browse/experimental/EdgeWordWordConsineSimilarityMain.java diff --git a/code/features/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResult.java b/code/features-search/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResult.java similarity index 100% rename from code/features/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResult.java rename to code/features-search/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResult.java diff --git a/code/features/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResultSet.java b/code/features-search/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResultSet.java similarity index 100% rename from code/features/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResultSet.java rename to code/features-search/random-websites/src/main/java/nu/marginalia/browse/model/BrowseResultSet.java diff --git a/code/features/random-websites/src/test/java/nu/marginalia/experimental/AndCardIntSetTest.java b/code/features-search/random-websites/src/test/java/nu/marginalia/experimental/AndCardIntSetTest.java similarity index 100% rename from code/features/random-websites/src/test/java/nu/marginalia/experimental/AndCardIntSetTest.java rename to code/features-search/random-websites/src/test/java/nu/marginalia/experimental/AndCardIntSetTest.java diff --git a/code/features/readme.md b/code/features-search/readme.md similarity index 73% rename from code/features/readme.md rename to code/features-search/readme.md index ddd34723..a3418afb 100644 --- a/code/features/readme.md +++ b/code/features-search/readme.md @@ -1,14 +1,9 @@ -# Features +# Search Features These are bits of search-engine related code that are relatively isolated pieces of business logic, that benefit from the clarity of being kept separate from the rest of the search engine code. -* [domain-ranking](domain-ranking/) contains ranking algorithms. * [query-parser](query-parser/) contains code for parsing the user-facing query grammar. - -* [adblock](adblock/) -* [pubdate](pubdate/) - * [screenshots](screenshots/) and [random-websites](random-websites/) contains SQL queries random -exploration mode. \ No newline at end of file + exploration mode. \ No newline at end of file diff --git a/code/features/screenshots/build.gradle b/code/features-search/screenshots/build.gradle similarity index 100% rename from code/features/screenshots/build.gradle rename to code/features-search/screenshots/build.gradle diff --git a/code/features/screenshots/src/main/java/nu/marginalia/screenshot/ScreenshotService.java b/code/features-search/screenshots/src/main/java/nu/marginalia/screenshot/ScreenshotService.java similarity index 100% rename from code/features/screenshots/src/main/java/nu/marginalia/screenshot/ScreenshotService.java rename to code/features-search/screenshots/src/main/java/nu/marginalia/screenshot/ScreenshotService.java diff --git a/code/libraries/LICENSE.txt b/code/libraries/LICENSE.txt new file mode 100644 index 00000000..2081662b --- /dev/null +++ b/code/libraries/LICENSE.txt @@ -0,0 +1,15 @@ +Copyright 2023 Viktor Lofgren + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +documentation files (the “Software”), to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of +the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/code/libraries/readme.md b/code/libraries/readme.md index af41df4a..7dabf9c9 100644 --- a/code/libraries/readme.md +++ b/code/libraries/readme.md @@ -1,6 +1,11 @@ # Libraries -These are libraries that are not strongly coupled to the search engine. +These are libraries that are not strongly coupled to the search engine's business logic. +These libraries may not depend on features, services, processes, models, etc. + +**NOTE:** These libraries are co-licensed under [the MIT license](LICENSE.txt). + +## Libraries * The [array](array/) library is for memory mapping large memory-areas, which Java has bad support for. It's designed to be able to easily replaced when *Java's Foreign Function And Memory API* is released. diff --git a/code/readme.md b/code/readme.md index 34a7c8c1..61266b6f 100644 --- a/code/readme.md +++ b/code/readme.md @@ -16,12 +16,29 @@ You'll find a short description in each module of what it does and how it relate * * [explorer](services-satellite/explorer-service) - [https://explore2.marginalia.nu/](https://explore2.marginalia.nu/) * an [internal API](api/) +### Features + +Features are relatively stand-alone components that serve some part of the domain. They aren't domain-independent, +but isolated. + +* [features-search](features-search) +* [features-crawl](features-crawl) +* [features-index](features-index) + + +### Processes + +* [crawl processes](crawl/) +* * [crawl-job-extractor](crawl/crawl-job-extractor-process) +* * [crawling-process](crawl/crawling-process) +* * [converting-process](crawl/converting-process) +* * [loading-process](crawl/loading-process) + ### Libraries and primitives + +Libraries are stand-alone code that is independent of the domain logic. + * [common](common/) elements for creating a service, a client etc. -* [index primitives](index/) -* [crawling and analysis](crawl/) * [libraries](libraries/) containing non-search specific code. * * [array](libraries/array/) - large memory mapped area library * * [btree](libraries/btree/) - static btree library -* + more -* [features](features/) containing code that is specific to search but shared among services. \ No newline at end of file diff --git a/code/services-core/assistant-service/build.gradle b/code/services-core/assistant-service/build.gradle index d9b04f0e..50821032 100644 --- a/code/services-core/assistant-service/build.gradle +++ b/code/services-core/assistant-service/build.gradle @@ -30,7 +30,7 @@ dependencies { implementation project(':code:common:service-discovery') implementation project(':code:common:service-client') - implementation project(':code:features:screenshots') + implementation project(':code:features-search:screenshots') implementation project(':code:libraries:language-processing') diff --git a/code/services-core/index-service/build.gradle b/code/services-core/index-service/build.gradle index 1ed30ea5..8b954153 100644 --- a/code/services-core/index-service/build.gradle +++ b/code/services-core/index-service/build.gradle @@ -30,13 +30,12 @@ dependencies { implementation project(':code:libraries:array') implementation project(':code:libraries:btree') - implementation project(':code:index:index-journal') - implementation project(':code:index:index-query') - implementation project(':code:index:index-forward') - implementation project(':code:index:index-reverse') - implementation project(':code:index:lexicon') - - implementation project(':code:features:domain-ranking') + implementation project(':code:features-index:index-journal') + implementation project(':code:features-index:index-query') + implementation project(':code:features-index:index-forward') + implementation project(':code:features-index:index-reverse') + implementation project(':code:features-index:lexicon') + implementation project(':code:features-index:domain-ranking') implementation libs.lombok diff --git a/code/services-core/index-service/readme.md b/code/services-core/index-service/readme.md index ff0e42da..4ff43a20 100644 --- a/code/services-core/index-service/readme.md +++ b/code/services-core/index-service/readme.md @@ -11,6 +11,8 @@ The index service knows which document contains which keywords. ## See Also -The index service relies heavily on the primitives in [index](../../index), -such as [index/index-forward](../../index/index-forward/), [index/index-reverse](../../index/index-reverse/), -and [index/index-query](../../index/index-query). +The index service relies heavily on the primitives in [features-index](../../features-index): + +* [features-index/index-forward](../../features-index/index-forward/) +* [features-index/index-reverse](../../features-index/index-reverse/) +* [features-index/index-query](../../features-index/index-query) \ No newline at end of file diff --git a/code/services-core/search-service/build.gradle b/code/services-core/search-service/build.gradle index 47e9f1d7..22f96602 100644 --- a/code/services-core/search-service/build.gradle +++ b/code/services-core/search-service/build.gradle @@ -24,7 +24,7 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:service') implementation project(':code:common:config') - implementation project(':code:index:index-query') + implementation project(':code:features-index:index-query') implementation project(':code:libraries:easy-lsh') implementation project(':code:libraries:language-processing') @@ -37,9 +37,9 @@ dependencies { implementation project(':code:common:service-client') implementation project(':code:common:renderer') - implementation project(':code:features:screenshots') - implementation project(':code:features:random-websites') - implementation project(':code:features:query-parser') + implementation project(':code:features-search:screenshots') + implementation project(':code:features-search:random-websites') + implementation project(':code:features-search:query-parser') implementation libs.lombok diff --git a/code/services-satellite/dating-service/build.gradle b/code/services-satellite/dating-service/build.gradle index 3208e477..85bdea4d 100644 --- a/code/services-satellite/dating-service/build.gradle +++ b/code/services-satellite/dating-service/build.gradle @@ -26,9 +26,9 @@ dependencies { implementation project(':code:common:service-discovery') implementation project(':code:common:service-client') implementation project(':code:common:renderer') - implementation project(':code:features:screenshots') + implementation project(':code:features-search:screenshots') + implementation project(':code:features-search:random-websites') implementation project(':code:libraries:language-processing') - implementation project(':code:features:random-websites') implementation libs.lombok diff --git a/code/services-satellite/dating-service/readme.md b/code/services-satellite/dating-service/readme.md index ba93439e..d011dbd0 100644 --- a/code/services-satellite/dating-service/readme.md +++ b/code/services-satellite/dating-service/readme.md @@ -7,5 +7,5 @@ Externally the service is available at [https://explore.marginalia.nu/](https:// # See Also -* [features/screenshots](../../features/screenshots) -* [features/random-websites](../../features/random-websites) \ No newline at end of file +* [features-search/screenshots](../../features-search/screenshots) +* [features-search/random-websites](../../features-search/random-websites) \ No newline at end of file diff --git a/code/services-satellite/explorer-service/build.gradle b/code/services-satellite/explorer-service/build.gradle index fee0e6dd..a18ff66e 100644 --- a/code/services-satellite/explorer-service/build.gradle +++ b/code/services-satellite/explorer-service/build.gradle @@ -27,7 +27,7 @@ dependencies { implementation project(':code:common:service-client') implementation project(':code:common:renderer') - implementation project(':code:features:random-websites') + implementation project(':code:features-search:random-websites') implementation libs.lombok diff --git a/code/services-satellite/explorer-service/readme.md b/code/services-satellite/explorer-service/readme.md index eab98e05..d7505197 100644 --- a/code/services-satellite/explorer-service/readme.md +++ b/code/services-satellite/explorer-service/readme.md @@ -6,5 +6,5 @@ Externally the service is available at [https://explore2.marginalia.nu/](https:/ # See Also -* [features/screenshots](../../features/screenshots) -* [features/random-websites](../../features/random-websites) \ No newline at end of file +* [features-search/screenshots](../../features-search/screenshots) +* [features-search/random-websites](../../features-search/random-websites) \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index db1822aa..e721ed2e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,23 +18,24 @@ include 'code:libraries:next-prime' include 'code:libraries:braille-block-punch-cards' include 'code:libraries:language-processing' -include 'code:features:screenshots' -include 'code:features:random-websites' -include 'code:features:domain-ranking' -include 'code:features:query-parser' -include 'code:features:adblock' -include 'code:features:pubdate' -include 'code:features:topic-detection' +include 'code:features-search:screenshots' +include 'code:features-search:random-websites' +include 'code:features-search:query-parser' + +include 'code:features-crawl:adblock' +include 'code:features-crawl:pubdate' +include 'code:features-crawl:topic-detection' include 'code:api:search-api' include 'code:api:index-api' include 'code:api:assistant-api' -include 'code:index:lexicon' -include 'code:index:index-journal' -include 'code:index:index-query' -include 'code:index:index-forward' -include 'code:index:index-reverse' +include 'code:features-index:lexicon' +include 'code:features-index:index-journal' +include 'code:features-index:index-query' +include 'code:features-index:index-forward' +include 'code:features-index:index-reverse' +include 'code:features-index:domain-ranking' include 'code:common:service-discovery' include 'code:common:service-client' @@ -44,14 +45,16 @@ include 'code:common:model' include 'code:common:renderer' include 'code:crawl:crawl-job-extractor-process' -include 'code:crawl:crawling-process' -include 'code:crawl:crawling-model' + include 'code:crawl:converting-process' -include 'code:crawl:converting-model' +include 'code:crawl:crawling-process' include 'code:crawl:loading-process' -include 'code:crawl:common' include 'code:crawl:experimental' +include 'code:crawl-models:converting-model' +include 'code:crawl-models:crawling-model' +include 'code:crawl-models:common' + include 'third-party:porterstemmer' include 'third-party:xz' include 'third-party:symspell' diff --git a/tools/screenshot/build.gradle b/tools/screenshot/build.gradle index 5a320cf2..899a5377 100644 --- a/tools/screenshot/build.gradle +++ b/tools/screenshot/build.gradle @@ -20,7 +20,7 @@ tasks.distZip.enabled = false dependencies { implementation project(':code:common:model') - implementation project(':code:features:screenshots') + implementation project(':code:features-search:screenshots') implementation project(':code:common:service') implementation libs.lombok