From f754b7f224f933a3bea9a8ea8e62a643dff0101e Mon Sep 17 00:00:00 2001 From: mattahmad Date: Sat, 27 Jul 2024 16:47:23 -0400 Subject: [PATCH 1/3] Proposed SatnogsTransferService to perform deduplication on Satnogs Telemetry API response field --- pom.xml | 15 +++ .../oresat/uniclogs/links/SatnogsTmLink.java | 62 ++++++++- .../services/SatnogsTransferService.java | 122 ++++++++++++++++++ .../tctm/SatnogsCommandPostprocessor.java | 47 +++++++ .../oresat/uniclogs/tctm/SatnogsPacket.java | 53 ++++++++ .../tctm/SatnogsPacketPreprocessor.java | 31 +++++ src/main/yamcs/etc/processor.yaml | 2 +- src/main/yamcs/etc/yamcs.oresat0.yaml | 25 +++- src/main/yamcs/mdb/oresat.xml | 10 ++ 9 files changed, 361 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/oresat/uniclogs/services/SatnogsTransferService.java create mode 100644 src/main/java/org/oresat/uniclogs/tctm/SatnogsCommandPostprocessor.java create mode 100644 src/main/java/org/oresat/uniclogs/tctm/SatnogsPacket.java create mode 100644 src/main/java/org/oresat/uniclogs/tctm/SatnogsPacketPreprocessor.java diff --git a/pom.xml b/pom.xml index ee76504..5ec414e 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,21 @@ commons-codec 1.15 + + com.fasterxml.jackson.core + jackson-core + 2.15.2 + + + com.fasterxml.jackson.core + jackson-databind + 2.15.2 + + + com.fasterxml.jackson.core + jackson-annotations + 2.15.2 + diff --git a/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java b/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java index 6e64fdf..ab8fa7b 100644 --- a/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java +++ b/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java @@ -1,9 +1,23 @@ package org.oresat.uniclogs.links; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.yamcs.ConfigurationException; +import org.yamcs.YConfiguration; +import org.yamcs.YamcsServer; import org.yamcs.tctm.AbstractTmDataLink; +import org.oresat.uniclogs.services.SatnogsTransferService; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; + public class SatnogsTmLink extends AbstractTmDataLink { + SatnogsTransferService tsfrService; + @Override protected Status connectionStatus() { // TODO Auto-generated method stub @@ -12,14 +26,54 @@ protected Status connectionStatus() { @Override protected void doStart() { - // TODO Auto-generated method stub - + + try { + + URL urlArtifacts = new URL("https://db.satnogs.org/api/telemetry/?app_source=&end=&format=json&is_decoded=&observer=&sat_id=&satellite=52017&start=&transmitter="); + HttpURLConnection conArtifacts = (HttpURLConnection) urlArtifacts.openConnection(); + conArtifacts.setRequestProperty("Authorization", "Bearer " + System.getenv("SATNOGSDB_AUTH_TOKEN")); + conArtifacts.setRequestMethod("GET"); + conArtifacts.setRequestProperty("contentType", "application/json; charset=utf-8"); + + BufferedReader inArtifacts = new BufferedReader( + new InputStreamReader(conArtifacts.getInputStream())); + String inputLineArtifacts; + StringBuilder contentArtifacts = new StringBuilder(); + while ((inputLineArtifacts = inArtifacts.readLine()) != null) { + contentArtifacts.append(inputLineArtifacts); + } + inArtifacts.close(); + + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode = objectMapper.readTree(String.valueOf(contentArtifacts)); + + // Access specific fields from the JSON structure + String contentArtifactsSet = jsonNode.get("results").asText(); + + tsfrService.getDeduplication(contentArtifactsSet); + + } catch (IOException e) { + this.log.error(e.getMessage()); + } } @Override protected void doStop() { // TODO Auto-generated method stub - + + } + + public SatnogsTmLink(String instanceName, YConfiguration config) { + String envName = config.getString("envService"); + this.tsfrService = YamcsServer.getServer().getInstance(instanceName).getService(SatnogsTransferService.class, envName); + + if (this.tsfrService == null) { + throw new ConfigurationException("Service " + envName + " does not exist or is not of class SatnogsTransferService."); + } + } + + public SatnogsTmLink(String instanceName) { + this(instanceName, YConfiguration.emptyConfig()); } - + } diff --git a/src/main/java/org/oresat/uniclogs/services/SatnogsTransferService.java b/src/main/java/org/oresat/uniclogs/services/SatnogsTransferService.java new file mode 100644 index 0000000..1ece3bb --- /dev/null +++ b/src/main/java/org/oresat/uniclogs/services/SatnogsTransferService.java @@ -0,0 +1,122 @@ +package org.oresat.uniclogs.services; + +import org.yamcs.AbstractYamcsService; +import org.yamcs.InitException; +import org.yamcs.YConfiguration; +import org.yamcs.utils.ByteArrayUtils; +import org.yamcs.yarch.Bucket; +import org.yamcs.yarch.YarchDatabase; +import org.yamcs.yarch.YarchDatabaseInstance; +import java.util.HashSet; +import java.util.Set; + +import java.io.IOException; + +public class SatnogsTransferService extends AbstractYamcsService { + static final String sequenceNumberId = "seqNum"; + static final String hmacKeyId = "hmacKey"; + String hmacEnvVar; + byte[] hmacKey; + Integer seqNum; + Bucket db; + + private byte[] loadHmacFromEnv(String varName) { + return System.getenv(varName).getBytes(); + } + + private Set processedData; + // processedData set keeps track of unique data + + @Override + protected void doStart() { + try { + this.hmacKey = this.loadHmacKey(); + this.seqNum = this.loadSeqNum(); + this.log.info("Loaded Sequence Number: " + this.seqNum); + processedData = new HashSet<>(); + } catch (IOException e) { + this.log.error(e.getMessage()); + } + this.notifyStarted(); + } + + @Override + protected void doStop() { + try { + this.saveSeqNum(this.seqNum); + this.saveHmacKey(this.hmacKey); + } catch (IOException e) { + this.log.error(e.getMessage()); + } + this.notifyStopped(); + } + + public Integer getSeqNum() { + Integer num = this.seqNum; + this.seqNum++; + return num; + } + + public Boolean getDeduplication(String contentArtifactsSet) { + // Check if the data is unique (not already processed) + if (!processedData.contains(contentArtifactsSet)){ + System.out.println("Processing unique data: " + contentArtifactsSet); + // Add the processed data to the set + this.log.info("DB Satnogs Artifacts: " + processedData.add(contentArtifactsSet)); + return true; + } else { + System.out.println("Duplicate data received: " + contentArtifactsSet); + // Handle duplicate data (If the text is present, it won't be added and the method returns false) + return false; + } + } + + public byte[] getHmacKey() { + return this.hmacKey; + } + + @Override + public void init(String yamcsInstance, String serviceName, YConfiguration config) throws InitException { + super.init(yamcsInstance, serviceName, config); + this.hmacEnvVar = config.getString("hmacEnvVar"); + log.info(this.hmacEnvVar); + YarchDatabaseInstance instanceDb = YarchDatabase.getInstance(yamcsInstance); + try { + this.db = instanceDb.getBucket("env"); + if (this.db == null) { + this.log.info(String.format("Env not found for %s, creating new env...", yamcsInstance)); + this.db = instanceDb.createBucket("env"); + } + } catch (IOException e) { + throw new InitException(e.getMessage()); + } + } + + private Integer loadSeqNum() throws IOException { + byte[] numBytes = db.getObject(sequenceNumberId); + if (numBytes == null) { + this.log.info(String.format("Sequence Number not found for %s, Sequence Number set to 1", yamcsInstance)); + this.saveSeqNum(1); + return 1; + } + return ByteArrayUtils.decodeInt(db.getObject(sequenceNumberId), 0); + } + + private byte[] loadHmacKey() throws IOException { + byte[] hmac = this.db.getObject(hmacKeyId); + if (hmac == null) { + this.log.info(String.format("Hmac Key not in %s. Loading from env var: %s", yamcsInstance, this.hmacEnvVar)); + hmac = this.loadHmacFromEnv(this.hmacEnvVar); + this.saveHmacKey(hmac); + } + return hmac; + } + + private void saveSeqNum(Integer seq) throws IOException { + db.putObject(sequenceNumberId, "Integer", null, ByteArrayUtils.encodeInt(seq)); + } + + private void saveHmacKey(byte[] hmacKey) throws IOException { + db.putObject(hmacKeyId, "bytes", null, hmacKey); + } +} \ No newline at end of file diff --git a/src/main/java/org/oresat/uniclogs/tctm/SatnogsCommandPostprocessor.java b/src/main/java/org/oresat/uniclogs/tctm/SatnogsCommandPostprocessor.java new file mode 100644 index 0000000..d567da7 --- /dev/null +++ b/src/main/java/org/oresat/uniclogs/tctm/SatnogsCommandPostprocessor.java @@ -0,0 +1,47 @@ +package org.oresat.uniclogs.tctm; +import org.oresat.uniclogs.services.SatnogsTransferService; +import org.yamcs.ConfigurationException; +import org.yamcs.YConfiguration; +import org.yamcs.YamcsServer; +import org.yamcs.cmdhistory.CommandHistoryPublisher; +import org.yamcs.commanding.PreparedCommand; +import org.yamcs.tctm.CommandPostprocessor; + + +public class SatnogsCommandPostprocessor implements CommandPostprocessor { + protected CommandHistoryPublisher cmdHistory; + SatnogsTransferService env; + + @Override + public byte[] process(PreparedCommand pc) { + Integer seqNum = this.env.getSeqNum(); + byte[] hmacKey = this.env.getHmacKey(); + + EDLPacket cmd = new EDLPacket(pc.getBinary(), seqNum, hmacKey); + byte[] cmdBin = cmd.getBinary(); + + pc.setBinary(cmdBin); + this.cmdHistory.publish(pc.getCommandId(), "edl-seqnum", seqNum); + this.cmdHistory.publish(pc.getCommandId(), PreparedCommand.CNAME_BINARY, cmdBin); + return cmdBin; + } + + @Override + public void setCommandHistoryPublisher(CommandHistoryPublisher commandHistoryListener) { + this.cmdHistory = commandHistoryListener; + } + + public SatnogsCommandPostprocessor(String instanceName, YConfiguration config) { + String envName = config.getString("envService"); + this.env = YamcsServer.getServer().getInstance(instanceName).getService(SatnogsTransferService.class, envName); + + if (this.env == null) { + throw new ConfigurationException("Service " + envName + " does not exist or is not of class SatnogsTransferService."); + } + } + + public SatnogsCommandPostprocessor(String instanceName) { + this(instanceName, YConfiguration.emptyConfig()); + } + +} diff --git a/src/main/java/org/oresat/uniclogs/tctm/SatnogsPacket.java b/src/main/java/org/oresat/uniclogs/tctm/SatnogsPacket.java new file mode 100644 index 0000000..990d339 --- /dev/null +++ b/src/main/java/org/oresat/uniclogs/tctm/SatnogsPacket.java @@ -0,0 +1,53 @@ +package org.oresat.uniclogs.tctm; + +import java.util.Arrays; + +import org.apache.commons.codec.digest.HmacAlgorithms; +import org.apache.commons.codec.digest.HmacUtils; +import org.yamcs.TmPacket; +import org.yamcs.xtce.util.HexUtils; + + +public class SatnogsPacket extends Packet { + private static final Integer SEQ_NUM_OFFSET = 7; + + public SatnogsPacket(byte[] packet, Integer seqNum, byte[] hmacSecret) { + //sequence number offset of 7 + super(packet, packet.length+36, seqNum, SEQ_NUM_OFFSET); + + // set sequence number in packet + this.encodeSeqNum(); + + + // set frame length in packet: C = (Total Number of Octets in the Transfer Frame) − 1 + // CRC adds 4, HMAC adds 32 -> (size + (36 - 1)) + this.encodeFrameLength(35, 4); + this.addHmac(hmacSecret); + + // Add CRC data to packet + this.encodeCrc(); + + } + + private void addHmac(byte[] hmacSecret) { + byte[] hmac = new HmacUtils(HmacAlgorithms.HMAC_SHA_256, hmacSecret).hmac(this.data.array()); + log.info(String.format("Hmac bytes: %s", Arrays.toString(hmac))); + this.data.put(hmac); + log.info(String.format("HMAC_SHA_256 (%s) added to packet (seqNum: %d).", HexUtils.hex(hmac), this.sequenceNumber)); + } + + protected void encodeCrc() { + int crc = this.calcCrc(this.data.array()); + log.info(String.format("CRC_32 (%d) added to packet (seqNum: %d).", crc, this.sequenceNumber)); + this.data.putInt(crc); + } + + public SatnogsPacket(TmPacket tmPacket) { + super(tmPacket.getPacket(), tmPacket.getPacket().length, getSequenceNumber(tmPacket.getPacket(), SEQ_NUM_OFFSET), SEQ_NUM_OFFSET); + } + + @Override + boolean validCrc() { + return crc32(0, this.data.array().length); + } +} diff --git a/src/main/java/org/oresat/uniclogs/tctm/SatnogsPacketPreprocessor.java b/src/main/java/org/oresat/uniclogs/tctm/SatnogsPacketPreprocessor.java new file mode 100644 index 0000000..75da3a2 --- /dev/null +++ b/src/main/java/org/oresat/uniclogs/tctm/SatnogsPacketPreprocessor.java @@ -0,0 +1,31 @@ +package org.oresat.uniclogs.tctm; + +import java.util.Date; + +import org.yamcs.TmPacket; +import org.yamcs.YConfiguration; +import org.yamcs.tctm.AbstractPacketPreprocessor; + +public class SatnogsPacketPreprocessor extends AbstractPacketPreprocessor { + + public SatnogsPacketPreprocessor(String yamcsInstance, YConfiguration config) { + super(yamcsInstance, config); + } + + public SatnogsPacketPreprocessor(String yamcsInstance) { + super(yamcsInstance, YConfiguration.emptyConfig()); + } + + @Override + public TmPacket process(TmPacket tmPacket) { + SatnogsPacket packet = new SatnogsPacket(tmPacket); + tmPacket.setSequenceCount(packet.getSeqNum()); + if (!packet.validCrc()) { + tmPacket.setInvalid(); + this.eventProducer.sendWarning("PACKET_CORRUPT", "Satnogs Response Packet Corrupted"); + } + tmPacket.setGenerationTime(new Date().getTime()); + tmPacket.setLocalGenTimeFlag(); + return tmPacket; + } +} \ No newline at end of file diff --git a/src/main/yamcs/etc/processor.yaml b/src/main/yamcs/etc/processor.yaml index f42b25a..b742c79 100644 --- a/src/main/yamcs/etc/processor.yaml +++ b/src/main/yamcs/etc/processor.yaml @@ -2,7 +2,7 @@ realtime: services: - class: org.yamcs.StreamTmPacketProvider args: - streams: ["edl_tm_realtime", "beacon_tm_realtime", "tm_dump"] + streams: ["edl_tm_realtime", "beacon_tm_realtime", "tm_dump", "satnogs_tm_realtime"] - class: org.yamcs.StreamTcCommandReleaser - class: org.yamcs.tctm.StreamParameterProvider - class: org.yamcs.algorithms.AlgorithmManager diff --git a/src/main/yamcs/etc/yamcs.oresat0.yaml b/src/main/yamcs/etc/yamcs.oresat0.yaml index ff14556..282dd34 100644 --- a/src/main/yamcs/etc/yamcs.oresat0.yaml +++ b/src/main/yamcs/etc/yamcs.oresat0.yaml @@ -1,7 +1,10 @@ services: - class: org.oresat.uniclogs.services.UniclogsEnvironment args: - hmacEnvVar: HMAC_KEY + hmacEnvVar: HMAC_KEY + - class: org.oresat.uniclogs.services.SatnogsTransferService + args: + hmacEnvVar: HMAC_KEY - class: org.yamcs.archive.XtceTmRecorder - class: org.yamcs.archive.ParameterRecorder - class: org.yamcs.archive.AlarmRecorder @@ -59,6 +62,22 @@ dataLinks: port: 10016 packetPreprocessorClassName: org.oresat.uniclogs.tctm.EdlPacketPreprocessor + - name: satnogs-tc-realtime + class: org.yamcs.tctm.UdpTcDataLink + stream: satnogs_tc_realtime + host: localhost + port: 10017 + commandPostprocessorClassName: org.oresat.uniclogs.tctm.SatnogsCommandPostprocessor + commandPostprocessorArgs: + envService: SatnogsTransferService + + - name: satnogs-tm-realtime + class: org.yamcs.tctm.UdpTmDataLink + stream: satnogs_tm_realtime + host: localhost + port: 10018 + packetPreprocessorClassName: org.oresat.uniclogs.tctm.SatnogsPacketPreprocessor + mdb: # Configuration of the active loaders # Valid loaders are: sheet, xtce or fully qualified name of the class @@ -93,6 +112,8 @@ streamConfig: rootContainer: "/OreSat/Beacon_Packet" - name: "edl_tm_realtime" rootContainer: "/OreSat/EDL_Packet" + - name: "satnogs_tm_realtime" + rootContainer: "/OreSat/Satnogs_Packet" - name: "tm_dump" cmdHist: ["cmdhist_realtime", "cmdhist_dump"] event: ["events_realtime", "events_dump"] @@ -101,3 +122,5 @@ streamConfig: tc: - name: "tc_realtime" processor: "realtime" + - name: "satnogs_tc_realtime" + processor: "realtime" diff --git a/src/main/yamcs/mdb/oresat.xml b/src/main/yamcs/mdb/oresat.xml index 00fc392..176097d 100644 --- a/src/main/yamcs/mdb/oresat.xml +++ b/src/main/yamcs/mdb/oresat.xml @@ -286,6 +286,16 @@ + + + + + 0 + + + + + From b27793d4a0adfd891e47a7e3cf6ba779133bb8d7 Mon Sep 17 00:00:00 2001 From: Matt Ahmad <21248577+mattahmad@users.noreply.github.com> Date: Sun, 18 Aug 2024 19:38:52 -0400 Subject: [PATCH 2/3] Update SatnogsTmLink.java --- src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java b/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java index ab8fa7b..704c30a 100644 --- a/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java +++ b/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java @@ -29,7 +29,7 @@ protected void doStart() { try { - URL urlArtifacts = new URL("https://db.satnogs.org/api/telemetry/?app_source=&end=&format=json&is_decoded=&observer=&sat_id=&satellite=52017&start=&transmitter="); + URL urlArtifacts = new URL("https://db.satnogs.org/api/telemetry/?app_source=&end=&format=json&is_decoded=&observer=&sat_id=&satellite=98867&start=&transmitter="); HttpURLConnection conArtifacts = (HttpURLConnection) urlArtifacts.openConnection(); conArtifacts.setRequestProperty("Authorization", "Bearer " + System.getenv("SATNOGSDB_AUTH_TOKEN")); conArtifacts.setRequestMethod("GET"); From 279cf82982fe4318a93b12e6c2c5494183f14627 Mon Sep 17 00:00:00 2001 From: Matt Ahmad <21248577+mattahmad@users.noreply.github.com> Date: Wed, 21 Aug 2024 07:51:27 -0400 Subject: [PATCH 3/3] Update SatnogsTmLink.java Update SatNOGS endpoint to https://db.satnogs.org/api/telemetry/?format=json&satellite=98867 --- src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java b/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java index 704c30a..1581533 100644 --- a/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java +++ b/src/main/java/org/oresat/uniclogs/links/SatnogsTmLink.java @@ -29,7 +29,7 @@ protected void doStart() { try { - URL urlArtifacts = new URL("https://db.satnogs.org/api/telemetry/?app_source=&end=&format=json&is_decoded=&observer=&sat_id=&satellite=98867&start=&transmitter="); + URL urlArtifacts = new URL("https://db.satnogs.org/api/telemetry/?format=json&satellite=98867"); HttpURLConnection conArtifacts = (HttpURLConnection) urlArtifacts.openConnection(); conArtifacts.setRequestProperty("Authorization", "Bearer " + System.getenv("SATNOGSDB_AUTH_TOKEN")); conArtifacts.setRequestMethod("GET");