161sh/CatgirlIntelligenceAgency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Retire defunct SMHI weather forecast integration.

Browse Source
This commit is contained in:
Viktor Lofgren 2023-01-30 13:25:41 +01:00
parent 4c2f54593e
commit 8168d512b8
22 changed files with 0 additions and 850 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/ServiceDescriptor.java
View File

@ -13,7 +13,6 @@ import nu.marginalia.wmsa.memex.MemexMain;
import nu.marginalia.wmsa.podcasts.PodcastScraperMain;
import nu.marginalia.wmsa.renderer.RendererMain;
import nu.marginalia.wmsa.resource_store.ResourceStoreMain;
import nu.marginalia.wmsa.smhi.scraper.SmhiScraperMain;
import org.apache.logging.log4j.core.lookup.MainMapLookup;
import java.util.Map;
@ -26,7 +25,6 @@ public enum ServiceDescriptor {
AUTH("auth", 5003, AuthMain.class),
API("api", 5004, ApiMain.class),
SMHI_SCRAPER("smhi-scraper",5012, SmhiScraperMain.class),
PODCST_SCRAPER("podcast-scraper", 5013, PodcastScraperMain.class),
EDGE_INDEX("edge-index", 5021, EdgeIndexMain.class),

9
marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/RendererService.java
View File

@ -1,22 +1,15 @@
package nu.marginalia.wmsa.renderer;
import com.google.gson.Gson;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import nu.marginalia.wmsa.client.GsonFactory;
import nu.marginalia.wmsa.configuration.server.Initialization;
import nu.marginalia.wmsa.configuration.server.MetricsServer;
import nu.marginalia.wmsa.configuration.server.Service;
import nu.marginalia.wmsa.resource_store.ResourceStoreClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RendererService extends Service {
private final Logger logger = LoggerFactory.getLogger(getClass());
private final Gson gson = GsonFactory.get();
private final ResourceStoreClient resourceStoreClient;
@ -24,7 +17,6 @@ public class RendererService extends Service {
public RendererService(ResourceStoreClient resourceStoreClient,
@Named("service-host") String ip,
@Named("service-port") Integer port,
SmhiRendererService smhiRendererService,
PodcastRendererService podcastRendererService,
StatusRendererService statusRendererService,
Initialization initialization,
@ -34,7 +26,6 @@ public class RendererService extends Service {
this.resourceStoreClient = resourceStoreClient;
smhiRendererService.start();
podcastRendererService.start();
statusRendererService.start();
}

82
marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/SmhiRendererService.java
View File

@ -1,82 +0,0 @@
package nu.marginalia.wmsa.renderer;
import com.google.gson.Gson;
import com.google.inject.Inject;
import lombok.SneakyThrows;
import nu.marginalia.wmsa.client.GsonFactory;
import nu.marginalia.wmsa.configuration.server.Context;
import nu.marginalia.wmsa.renderer.mustache.MustacheRenderer;
import nu.marginalia.wmsa.renderer.mustache.RendererFactory;
import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiIndexReq;
import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiPrognosReq;
import nu.marginalia.wmsa.resource_store.ResourceStoreClient;
import nu.marginalia.wmsa.resource_store.model.RenderedResource;
import nu.marginalia.wmsa.smhi.model.PrognosData;
import nu.marginalia.wmsa.smhi.model.index.IndexPlatser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import spark.Request;
import spark.Response;
import spark.Spark;
import java.time.LocalDateTime;
import java.util.concurrent.TimeUnit;
public class SmhiRendererService {
private final Logger logger = LoggerFactory.getLogger(getClass());
private final Gson gson = GsonFactory.get();
private final RendererFactory rendererFactory = new RendererFactory();
private final MustacheRenderer<IndexPlatser> indexRenderer;
private final MustacheRenderer<PrognosData> prognosRenderer;
private final ResourceStoreClient resourceStoreClient;
@Inject @SneakyThrows
public SmhiRendererService(ResourceStoreClient resourceStoreClient) {
this.resourceStoreClient = resourceStoreClient;
indexRenderer = rendererFactory.renderer( "smhi/index");
prognosRenderer = rendererFactory.renderer( "smhi/prognos");
}
public void start() {
Spark.post("/render/smhi/index", this::renderSmhiIndex);
Spark.post("/render/smhi/prognos", this::renderSmhiPrognos);
}
private Object renderSmhiIndex(Request request, Response response) {
var requestText = request.body();
var req = gson.fromJson(requestText, RenderSmhiIndexReq.class);
logger.info("renderSmhiIndex()");
var resource = new RenderedResource("index.html",
LocalDateTime.MAX,
indexRenderer.render(new IndexPlatser(req.platser)));
resourceStoreClient.putResource(Context.fromRequest(request), "smhi", resource)
.timeout(10, TimeUnit.SECONDS)
.blockingSubscribe();
return "";
}
private Object renderSmhiPrognos(Request request, Response response) {
var requestText = request.body();
var req = gson.fromJson(requestText, RenderSmhiPrognosReq.class);
logger.info("renderSmhiPrognos({})", req.data.plats.namn);
var resource = new RenderedResource(req.data.plats.getUrl(),
LocalDateTime.now().plusHours(3),
prognosRenderer.render(req.data));
resourceStoreClient.putResource(Context.fromRequest(request), "smhi", resource)
.timeout(10, TimeUnit.SECONDS)
.blockingSubscribe();
return "";
}
}

15
marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/client/RendererClient.java
View File

@ -11,8 +11,6 @@ import nu.marginalia.wmsa.podcasts.model.Podcast;
import nu.marginalia.wmsa.podcasts.model.PodcastEpisode;
import nu.marginalia.wmsa.podcasts.model.PodcastListing;
import nu.marginalia.wmsa.podcasts.model.PodcastNewEpisodes;
import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiIndexReq;
import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiPrognosReq;
import javax.inject.Inject;
import java.util.concurrent.TimeUnit;
@ -24,19 +22,6 @@ public class RendererClient extends AbstractDynamicClient{
super(ServiceDescriptor.RENDERER);
}
@SneakyThrows
public Observable<HttpStatusCode> render(Context ctx, RenderSmhiPrognosReq req) {
return post(ctx, "/render/smhi/prognos", req)
.timeout(5, TimeUnit.SECONDS, Observable.error(new TimeoutException("RendererClient.renderSmhiPrognos()")));
}
@SneakyThrows
public Observable<HttpStatusCode> render(Context ctx, RenderSmhiIndexReq req) {
return post(ctx, "/render/smhi/index", req)
.timeout(5, TimeUnit.SECONDS, Observable.error(new TimeoutException("RendererClient.renderSmhiIndex()")));
}
@SneakyThrows
public Observable<HttpStatusCode> render(Context ctx, PodcastNewEpisodes req) {
return post(ctx, "/render/podcast/new", req)

13
marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/request/smhi/RenderSmhiIndexReq.java
View File

@ -1,13 +0,0 @@
package nu.marginalia.wmsa.renderer.request.smhi;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import nu.marginalia.wmsa.smhi.model.Plats;
import java.util.List;
@NoArgsConstructor @AllArgsConstructor @Getter
public class RenderSmhiIndexReq {
public List<Plats> platser;
}

11
marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/request/smhi/RenderSmhiPrognosReq.java
View File

@ -1,11 +0,0 @@
package nu.marginalia.wmsa.renderer.request.smhi;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import nu.marginalia.wmsa.smhi.model.PrognosData;
@NoArgsConstructor @AllArgsConstructor @Getter
public class RenderSmhiPrognosReq {
public PrognosData data;
}

79
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/SmhiScraperService.java
View File

@ -1,79 +0,0 @@
package nu.marginalia.wmsa.smhi;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import io.reactivex.rxjava3.schedulers.Schedulers;
import nu.marginalia.wmsa.configuration.server.Context;
import nu.marginalia.wmsa.configuration.server.Initialization;
import nu.marginalia.wmsa.configuration.server.MetricsServer;
import nu.marginalia.wmsa.configuration.server.Service;
import nu.marginalia.wmsa.renderer.client.RendererClient;
import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiIndexReq;
import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiPrognosReq;
import nu.marginalia.wmsa.smhi.model.Plats;
import nu.marginalia.wmsa.smhi.model.PrognosData;
import nu.marginalia.wmsa.smhi.scraper.crawler.SmhiCrawler;
import nu.marginalia.wmsa.smhi.scraper.crawler.entity.SmhiEntityStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import spark.Spark;
import java.util.Comparator;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
public class SmhiScraperService extends Service {
private final SmhiCrawler crawler;
private final SmhiEntityStore entityStore;
private final RendererClient rendererClient;
private final Logger logger = LoggerFactory.getLogger(getClass());
private final Initialization initialization;
@Inject
public SmhiScraperService(@Named("service-host") String ip,
@Named("service-port") Integer port,
SmhiCrawler crawler,
SmhiEntityStore entityStore,
RendererClient rendererClient,
Initialization initialization,
MetricsServer metricsServer) {
super(ip, port, initialization, metricsServer);
this.crawler = crawler;
this.entityStore = entityStore;
this.rendererClient = rendererClient;
this.initialization = initialization;
Spark.awaitInitialization();
Schedulers.newThread().scheduleDirect(this::start);
}
private void start() {
initialization.waitReady();
rendererClient.waitReady();
entityStore.platser.debounce(6, TimeUnit.SECONDS)
.subscribe(this::updateIndex);
entityStore.prognosdata.subscribe(this::updatePrognos);
crawler.start();
}
private void updatePrognos(PrognosData prognosData) {
rendererClient
.render(Context.internal(), new RenderSmhiPrognosReq(prognosData))
.timeout(30, TimeUnit.SECONDS)
.blockingSubscribe();
}
private void updateIndex(Plats unused) {
var platser = entityStore.platser().stream()
.sorted(Comparator.comparing(plats -> plats.namn))
.collect(Collectors.toList());
rendererClient
.render(Context.internal(), new RenderSmhiIndexReq(platser))
.timeout(30, TimeUnit.SECONDS)
.blockingSubscribe();
}
}

9
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Parameter.java
View File

@ -1,9 +0,0 @@
package nu.marginalia.wmsa.smhi.model;
public class Parameter {
public String name;
public String levelType;
public String level;
public String unit;
public String[] values;
}

42
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Plats.java
View File

@ -1,42 +0,0 @@
package nu.marginalia.wmsa.smhi.model;
import lombok.Getter;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
@Getter
public class Plats {
public final String namn;
public final double latitud;
public final double longitud;
public String getUrl() {
return namn.toLowerCase()+".html";
}
public Plats(String namn, String latitud, String longitud) {
this.namn = namn;
this.longitud = Double.parseDouble(longitud);
this.latitud = Double.parseDouble(latitud);
}
public String toString() {
return String.format("Plats[%s %s %s]", namn, longitud, latitud);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Plats plats = (Plats) o;
return new EqualsBuilder().append(latitud, plats.latitud).append(longitud, plats.longitud).append(namn, plats.namn).isEquals();
}
@Override
public int hashCode() {
return new HashCodeBuilder(17, 37).append(namn).append(latitud).append(longitud).toHashCode();
}
}

16
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Platser.java
View File

@ -1,16 +0,0 @@
package nu.marginalia.wmsa.smhi.model;
import java.util.List;
public class Platser {
private final List<Plats> platser;
public Platser(List<Plats> platser) {
this.platser = platser;
}
public List<Plats> getPlatser() {
return platser;
}
}

41
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/PrognosData.java
View File

@ -1,41 +0,0 @@
package nu.marginalia.wmsa.smhi.model;
import nu.marginalia.wmsa.smhi.model.dyn.Dygnsdata;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
public class PrognosData {
public final String crawlTime = LocalDateTime.now().toString();
public String approvedTime;
public String referenceTime;
public String expires;
public Plats plats;
public final List<Tidpunkt> timeSeries = new ArrayList<>();
public String getBastFore() {
return LocalDateTime.parse(crawlTime).atZone(ZoneId.of("Europe/Stockholm"))
.plusHours(3)
.format(DateTimeFormatter.ISO_TIME);
}
public Plats getPlats() {
return plats;
}
public List<Tidpunkt> getTidpunkter() {
return timeSeries;
}
public List<Dygnsdata> getDygn() {
return timeSeries.stream().map(Tidpunkt::getDate).distinct()
.map(datum -> new Dygnsdata(datum, this))
.collect(Collectors.toList());
}
}

75
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Tidpunkt.java
View File

@ -1,75 +0,0 @@
package nu.marginalia.wmsa.smhi.model;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.temporal.ChronoField;
import java.util.ArrayList;
import java.util.List;
public class Tidpunkt {
private static final ZoneId serverZoneId = ZoneId.of("GMT");
private static final ZoneId localZoneId = ZoneId.of("Europe/Stockholm");
private static final DateTimeFormatter timeFormatter = (new DateTimeFormatterBuilder())
.appendValue(ChronoField.HOUR_OF_DAY, 2)
.appendLiteral(':')
.appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.toFormatter();
public String validTime;
public final List<Parameter> parameters = new ArrayList<>();
private String getParam(String name) {
var data = parameters.stream().filter(p -> name.equals(p.name)).map(p->p.values).findFirst().orElseGet(() -> new String[0]);
if (data.length > 0) {
return data[0];
}
return null;
}
public String getDate() {
return ZonedDateTime.parse(validTime).toLocalDateTime().atZone(serverZoneId).toOffsetDateTime().atZoneSameInstant(localZoneId).format(DateTimeFormatter.ISO_LOCAL_DATE);
}
public String getTime() {
return ZonedDateTime.parse(validTime).toLocalDateTime().atZone(serverZoneId).toOffsetDateTime().atZoneSameInstant(localZoneId).format(timeFormatter);
}
public String getTemp() {
return getParam("t");
}
public String getMoln() {
return getParam("tcc_mean");
}
public String getVind() {
return getParam("ws");
}
public String getByvind() {
return getParam("gust");
}
public String getNederbord() {
return getParam("pmedian");
}
public String getNederbordTyp() {
switch(getParam("pcat")) {
case "1": return "S";
case "2": return "SB";
case "3": return "R";
case "4": return "D";
case "5": return "UKR";
case "6": return "UKD";
default:
return "";
}
}
public String getVindRiktning() {
return getParam("wd");
}
public String toString() {
return String.format("Tidpunkt[%s %s]", validTime, getTemp());
}
}

40
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/dyn/Dygnsdata.java
View File

@ -1,40 +0,0 @@
package nu.marginalia.wmsa.smhi.model.dyn;
import nu.marginalia.wmsa.smhi.model.PrognosData;
import nu.marginalia.wmsa.smhi.model.Tidpunkt;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.stream.Collectors;
public class Dygnsdata {
public final String date;
private final PrognosData data;
public Dygnsdata(String date, PrognosData data) {
this.date = date;
this.data = data;
}
public String getDate() {
return date;
}
public List<Tidpunkt> getData() {
String d = getDate();
return data.timeSeries.stream().filter(p -> d.equals(p.getDate())).collect(Collectors.toList());
}
public String getVeckodag() {
switch (LocalDate.parse(date, DateTimeFormatter.ISO_LOCAL_DATE).getDayOfWeek()) {
case MONDAY: return "M&aring;ndag";
case TUESDAY: return "Tisdag";
case WEDNESDAY: return "Onsdag";
case THURSDAY: return "Torsdag";
case FRIDAY: return "Fredag";
case SATURDAY: return "L&ouml;rdag";
case SUNDAY: return "S&ouml;ndag";
}
return "Annandag";
}
}

13
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/index/IndexPlats.java
View File

@ -1,13 +0,0 @@
package nu.marginalia.wmsa.smhi.model.index;
import lombok.AllArgsConstructor;
import lombok.Getter;
import nu.marginalia.wmsa.smhi.model.Plats;
import java.util.List;
@Getter @AllArgsConstructor
public class IndexPlats {
String nyckel;
List<Plats> platser;
}

28
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/index/IndexPlatser.java
View File

@ -1,28 +0,0 @@
package nu.marginalia.wmsa.smhi.model.index;
import lombok.Getter;
import nu.marginalia.wmsa.smhi.model.Plats;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Getter
public class IndexPlatser {
final List<IndexPlats> platserPerNyckel = new ArrayList<>();
public IndexPlatser(List<Plats> platser) {
var platsMap = kategoriseraEfterNyckel(platser);
platsMap.keySet().stream().sorted()
.forEach(p -> platserPerNyckel.add(new IndexPlats(p, platsMap.get(p))));
}
private Map<String, List<Plats>> kategoriseraEfterNyckel(List<Plats> platser) {
return platser.stream().collect(
Collectors.groupingBy(p ->
p.namn.substring(0, 1)
.toUpperCase()));
}
}

44
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/PlatsReader.java
View File

@ -1,44 +0,0 @@
package nu.marginalia.wmsa.smhi.scraper;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import com.opencsv.CSVReader;
import nu.marginalia.wmsa.smhi.model.Plats;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@Singleton
public class PlatsReader {
private final String fileName;
@Inject
public PlatsReader(@Named("plats-csv-file") String fileName) {
this.fileName = fileName;
}
public List<Plats> readPlatser() throws Exception {
List<Plats> platser = new ArrayList<>();
var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream(fileName),
"Kunde inte ladda " + fileName);
try (var reader = new CSVReader(new InputStreamReader(resource, StandardCharsets.UTF_8))) {
for (;;) {
String[] strings = reader.readNext();
if (strings == null) {
return platser;
}
platser.add(skapaPlats(strings));
}
}
}
private Plats skapaPlats(String[] strings) {
return new Plats(strings[0], strings[1], strings[2]);
}
}

32
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/SmhiScraperMain.java
View File

@ -1,32 +0,0 @@
package nu.marginalia.wmsa.smhi.scraper;
import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import nu.marginalia.wmsa.configuration.MainClass;
import nu.marginalia.wmsa.configuration.ServiceDescriptor;
import nu.marginalia.wmsa.configuration.module.ConfigurationModule;
import nu.marginalia.wmsa.configuration.server.Initialization;
import nu.marginalia.wmsa.smhi.SmhiScraperService;
import java.io.IOException;
public class SmhiScraperMain extends MainClass {
private final SmhiScraperService service;
@Inject
public SmhiScraperMain(SmhiScraperService service) {
this.service = service;
}
public static void main(String... args) {
init(ServiceDescriptor.SMHI_SCRAPER, args);
Injector injector = Guice.createInjector(
new SmhiScraperModule(),
new ConfigurationModule());
injector.getInstance(SmhiScraperMain.class);
injector.getInstance(Initialization.class).setReady();
}
}

12
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/SmhiScraperModule.java
View File

@ -1,12 +0,0 @@
package nu.marginalia.wmsa.smhi.scraper;
import com.google.inject.AbstractModule;
import com.google.inject.name.Names;
public class SmhiScraperModule extends AbstractModule {
public void configure() {
bind(String.class).annotatedWith(Names.named("plats-csv-file")).toInstance("data/smhi/stader.csv");
bind(String.class).annotatedWith(Names.named("smhi-user-agent")).toInstance("kontakt@marginalia.nu");
}
}

88
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiBackendApi.java
View File

@ -1,88 +0,0 @@
package nu.marginalia.wmsa.smhi.scraper.crawler;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.name.Named;
import nu.marginalia.wmsa.smhi.model.Plats;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.Locale;
@Singleton
public class SmhiBackendApi {
private final Logger logger = LoggerFactory.getLogger(getClass());
private final String server = "https://opendata-download-metfcst.smhi.se/api";
private final PoolingHttpClientConnectionManager connectionManager;
private final String userAgent;
@Inject
public SmhiBackendApi(@Named("smhi-user-agent") String userAgent) {
this.userAgent = userAgent;
connectionManager = new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(200);
connectionManager.setDefaultMaxPerRoute(20);
HttpHost host = new HttpHost("https://opendata-download-metfcst.smhi.se");
connectionManager.setMaxPerRoute(new HttpRoute(host), 50);
}
public SmhiApiRespons hamtaData(Plats plats) throws Exception {
var client = HttpClients.custom()
.setConnectionManager(connectionManager)
.build();
String url = String.format(Locale.US, "%s/category/pmp3g/version/2/geotype/point/lon/%f/lat/%f/data.json",
server, plats.longitud, plats.latitud);
Thread.sleep(100);
logger.info("Fetching {} - {}", plats, url);
HttpGet get = new HttpGet(url);
get.addHeader("User-Agent", userAgent);
try (var rsp = client.execute(get)) {
var entity = rsp.getEntity();
String content = new String(entity.getContent().readAllBytes());
int statusCode = rsp.getStatusLine().getStatusCode();
var expires =
Arrays.stream(rsp.getHeaders("Expires"))
.map(Header::getValue)
.map(DateTimeFormatter.RFC_1123_DATE_TIME::parse)
.map(LocalDateTime::from)
.findFirst().map(Object::toString).orElse("");
if (statusCode == 200) {
return new SmhiApiRespons(content, expires, plats);
}
throw new IllegalStateException("Fel i backend " + statusCode + " " + content);
}
}
}
class SmhiApiRespons {
public final String jsonContent;
public final String expiryDate;
public final Plats plats;
SmhiApiRespons(String jsonContent, String expiryDate, Plats plats) {
this.jsonContent = jsonContent;
this.expiryDate = expiryDate;
this.plats = plats;
}
}

106
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiCrawler.java
View File

@ -1,106 +0,0 @@
package nu.marginalia.wmsa.smhi.scraper.crawler;
import com.google.gson.*;
import com.google.inject.Inject;
import io.reactivex.rxjava3.core.Maybe;
import io.reactivex.rxjava3.core.Observable;
import io.reactivex.rxjava3.disposables.Disposable;
import io.reactivex.rxjava3.schedulers.Schedulers;
import lombok.SneakyThrows;
import nu.marginalia.wmsa.smhi.model.Plats;
import nu.marginalia.wmsa.smhi.model.PrognosData;
import nu.marginalia.wmsa.smhi.scraper.PlatsReader;
import nu.marginalia.wmsa.smhi.scraper.crawler.entity.SmhiEntityStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.reflect.Type;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
public class SmhiCrawler {
private final Logger logger = LoggerFactory.getLogger(getClass());
private final Gson gson;
private final SmhiBackendApi api;
private final SmhiEntityStore store;
private final List<Plats> platser;
private Disposable job;
@Inject @SneakyThrows
public SmhiCrawler(SmhiBackendApi backendApi, SmhiEntityStore store, PlatsReader platsReader) {
this.api = backendApi;
this.store = store;
this.platser = platsReader.readPlatser();
class LocalDateAdapter implements JsonDeserializer<LocalDateTime> {
@Override
public LocalDateTime deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
return LocalDateTime
.parse(json.getAsString(), DateTimeFormatter.ISO_ZONED_DATE_TIME);
}
}
gson = new GsonBuilder()
.registerTypeAdapter(LocalDateTime.class, new LocalDateAdapter())
.create();
}
public void start() {
job = Observable
.fromIterable(new ArrayList<>(platser))
.subscribeOn(Schedulers.io())
.filter(this::isNeedsUpdate)
.take(5)
.flatMapMaybe(this::hamtaData)
.repeatWhen(this::repeatDelay)
.doOnError(this::handleError)
.subscribe(store::offer);
}
public void stop() {
Optional.ofNullable(job).ifPresent(Disposable::dispose);
}
private Observable<?> repeatDelay(Observable<Object> completed) {
return completed.delay(1, TimeUnit.SECONDS);
}
protected void handleError(Throwable throwable) {
logger.error("Caught error", throwable);
}
public Maybe<PrognosData> hamtaData(Plats plats) {
try {
var data = api.hamtaData(plats);
PrognosData model = gson.fromJson(data.jsonContent, PrognosData.class);
model.expires = data.expiryDate;
model.plats = plats;
return Maybe.just(model);
}
catch (Exception ex) {
logger.error("Failed to fetch data", ex);
return Maybe.empty();
}
}
boolean isNeedsUpdate(Plats plats) {
var prognos = store.prognos(plats);
if (null == prognos) {
return true;
}
LocalDateTime crawlTime = LocalDateTime.parse(prognos.crawlTime);
return crawlTime.plusHours(1).isBefore(LocalDateTime.now());
}
}

62
marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/entity/SmhiEntityStore.java
View File

@ -1,62 +0,0 @@
package nu.marginalia.wmsa.smhi.scraper.crawler.entity;
import com.google.inject.Singleton;
import io.reactivex.rxjava3.subjects.PublishSubject;
import nu.marginalia.wmsa.smhi.model.Plats;
import nu.marginalia.wmsa.smhi.model.PrognosData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@Singleton
public class SmhiEntityStore {
private final ReadWriteLock rwl = new ReentrantReadWriteLock();
private final Map<Plats, PrognosData> data = new HashMap<>();
public final PublishSubject<Plats> platser = PublishSubject.create();
public final PublishSubject<PrognosData> prognosdata = PublishSubject.create();
Logger logger = LoggerFactory.getLogger(getClass());
public boolean offer(PrognosData modell) {
Lock lock = this.rwl.writeLock();
try {
lock.lock();
if (data.put(modell.plats, modell) == null) {
platser.onNext(modell.plats);
}
prognosdata.onNext(modell);
}
finally {
lock.unlock();
}
return true;
}
public List<Plats> platser() {
Lock lock = this.rwl.readLock();
try {
lock.lock();
return new ArrayList<>(data.keySet());
}
finally {
lock.unlock();
}
}
public PrognosData prognos(Plats plats) {
Lock lock = this.rwl.readLock();
try {
lock.lock();
return data.get(plats);
}
finally {
lock.unlock();
}
}
}

31
marginalia_nu/src/test/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiBackendApiTest.java
View File

@ -1,31 +0,0 @@
package nu.marginalia.wmsa.smhi.scraper.crawler;
import nu.marginalia.wmsa.smhi.model.Plats;
import org.junit.jupiter.api.Test;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
class SmhiBackendApiTest {
@Test
void hamtaData() throws Exception {
var api = new SmhiBackendApi("nu.marginalia");
System.out.println(api.hamtaData(new Plats("Ystad", "55.42966", "13.82041"))
.jsonContent
);
}
@Test
public void testDatum() {
System.out.println(LocalDateTime.parse("2021-05-29T14:06:48Z",
DateTimeFormatter.ISO_ZONED_DATE_TIME)
.atZone(ZoneId.of("GMT"))
.toOffsetDateTime()
.atZoneSameInstant(ZoneId.of("Europe/Stockholm"))
);
}
}