(site-info) Try to discover the schema of the website with a site:-query

The site info view can't blindly assume that every website supports https.  To figure out which schema to use when linking to a site, execute a single-result search for site:domain.name and then grab the schema off the result.

To allow this, a count parameter is introduced to doSiteSearch() in SearchOperator.
This commit is contained in:
Viktor Lofgren 2023-12-16 16:34:53 +01:00
parent 8bbb533c9a
commit 320882c34a
4 changed files with 19 additions and 8 deletions

View File

@ -68,13 +68,15 @@ public class SearchOperator {
} }
public List<UrlDetails> doSiteSearch(Context ctx, public List<UrlDetails> doSiteSearch(Context ctx,
String domain) { String domain,
int count) {
var queryParams = paramFactory.forSiteSearch(domain); var queryParams = paramFactory.forSiteSearch(domain, count);
var queryResponse = queryClient.search(ctx, queryParams); var queryResponse = queryClient.search(ctx, queryParams);
return searchQueryService.getResultsFromQuery(queryResponse); return searchQueryService.getResultsFromQuery(queryResponse);
} }
public List<UrlDetails> doBacklinkSearch(Context ctx, public List<UrlDetails> doBacklinkSearch(Context ctx,
String domain) { String domain) {

View File

@ -36,7 +36,7 @@ public class SearchQueryParamFactory {
} }
public QueryParams forSiteSearch(String domain) { public QueryParams forSiteSearch(String domain, int count) {
return new QueryParams("site:"+domain, return new QueryParams("site:"+domain,
null, null,
List.of(), List.of(),
@ -48,12 +48,11 @@ public class SearchQueryParamFactory {
SpecificationLimit.none(), SpecificationLimit.none(),
SpecificationLimit.none(), SpecificationLimit.none(),
List.of(), List.of(),
new QueryLimits(100, 100, 100, 512), new QueryLimits(count, count, 100, 512),
SearchSetIdentifier.NONE SearchSetIdentifier.NONE
); );
} }
public QueryParams forBacklinkSearch(String domain) { public QueryParams forBacklinkSearch(String domain) {
return new QueryParams("links:"+domain, return new QueryParams("links:"+domain,
null, null,

View File

@ -119,6 +119,7 @@ public class SearchSiteInfoService {
final DomainInformation domainInfo; final DomainInformation domainInfo;
final List<SimilarDomain> similarSet; final List<SimilarDomain> similarSet;
final List<SimilarDomain> linkingDomains; final List<SimilarDomain> linkingDomains;
String url = "https://" + domainName + "/";;
if (domainId < 0 || !assistantClient.isAccepting()) { if (domainId < 0 || !assistantClient.isAccepting()) {
domainInfo = createDummySiteInfo(domainName); domainInfo = createDummySiteInfo(domainName);
@ -133,10 +134,16 @@ public class SearchSiteInfoService {
linkingDomains = assistantClient linkingDomains = assistantClient
.linkedDomains(ctx, domainId, 100) .linkedDomains(ctx, domainId, 100)
.blockingFirst(); .blockingFirst();
List<UrlDetails> sampleResults = searchOperator.doSiteSearch(ctx, domainName, 1);
if (!sampleResults.isEmpty()) {
url = sampleResults.getFirst().url.withPathAndParam("/", null).toString();
}
} }
return new SiteInfoWithContext(domainName, return new SiteInfoWithContext(domainName,
domainId, domainId,
url,
domainInfo, domainInfo,
similarSet, similarSet,
linkingDomains linkingDomains
@ -154,7 +161,7 @@ public class SearchSiteInfoService {
private Docs listDocs(Context ctx, String domainName) { private Docs listDocs(Context ctx, String domainName) {
return new Docs(domainName, return new Docs(domainName,
domainQueries.tryGetDomainId(new EdgeDomain(domainName)).orElse(-1), domainQueries.tryGetDomainId(new EdgeDomain(domainName)).orElse(-1),
searchOperator.doSiteSearch(ctx, domainName)); searchOperator.doSiteSearch(ctx, domainName, 100));
} }
public record Docs(Map<String, Boolean> view, public record Docs(Map<String, Boolean> view,
@ -190,11 +197,13 @@ public class SearchSiteInfoService {
Map<String, Boolean> domainState, Map<String, Boolean> domainState,
String domain, String domain,
long domainId, long domainId,
String siteUrl,
DomainInformation domainInformation, DomainInformation domainInformation,
List<SimilarDomain> similar, List<SimilarDomain> similar,
List<SimilarDomain> linking) { List<SimilarDomain> linking) {
public SiteInfoWithContext(String domain, public SiteInfoWithContext(String domain,
long domainId, long domainId,
String siteUrl,
DomainInformation domainInformation, DomainInformation domainInformation,
List<SimilarDomain> similar, List<SimilarDomain> similar,
List<SimilarDomain> linking List<SimilarDomain> linking
@ -204,6 +213,7 @@ public class SearchSiteInfoService {
Map.of(domainInfoState(domainInformation), true), Map.of(domainInfoState(domainInformation), true),
domain, domain,
domainId, domainId,
siteUrl,
domainInformation, domainInformation,
similar, similar,
linking); linking);

View File

@ -5,10 +5,10 @@
<div id="similar-view" data-layout="{{layout}}"> <div id="similar-view" data-layout="{{layout}}">
<div id="similar-info"> <div id="similar-info">
<h2><span title="External Link">&#x1F30E;</span>&nbsp;<a rel="external noopener" href="https://{{domain}}/">{{domain}}</a></h2> <h2><span title="External Link">&#x1F30E;</span>&nbsp;<a rel="external noopener" href="{{siteUrl}}">{{domain}}</a></h2>
<a rel="external noopener" href="https://{{domain}}/"> <a rel="external noopener" href="{{siteUrl}}">
<img class="screenshot" width="300" height="225" src="/screenshot/{{domainId}}" alt="Screenshot of {{domain}}" /> <img class="screenshot" width="300" height="225" src="/screenshot/{{domainId}}" alt="Screenshot of {{domain}}" />
</a> </a>
{{#with domainInformation}} {{#with domainInformation}}