Skip to content

Commit

Permalink
fixed: ARD: HD-Erkennung
Browse files Browse the repository at this point in the history 
closed #323
  • Loading branch information
@alex1702
alex1702 committed Dec 30, 2017
2 parents db2726f + 025672f commit 598db97
Show file tree
Hide file tree
Showing 8 changed files with 525 additions and 71 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ apply from: "${project.rootDir}/gradle/eclipse.gradle"
sourceCompatibility = 1.8
targetCompatibility = 1.8
group = 'de.mediathekview'
version = '3.1.42'
version = '3.1.43'

def jarName = 'MServer.jar'
def mainClass = 'mServer.Main'
Expand Down
160 changes: 92 additions & 68 deletions src/main/java/mServer/crawler/sender/ard/ArdVideoDeserializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import mServer.crawler.sender.newsearch.Qualities;

Expand All @@ -21,74 +23,122 @@ public class ArdVideoDeserializer implements JsonDeserializer<ArdVideoDTO> {
private static final String JSON_ELEMENT_WIDTH = "_width";
private static final String JSON_ELEMENT_HEIGHT = "_height";

private static String addMissingHttpPrefixIfNecessary(String aUrl) {
if(aUrl.startsWith("//")) {
return TEXT_START_HTTP + ":" + aUrl;
}

return aUrl;
}

@Override
public ArdVideoDTO deserialize(JsonElement aJsonElement, Type aType, JsonDeserializationContext aContext) throws JsonParseException {
ArdVideoDTO dto = new ArdVideoDTO();

if (aJsonElement.isJsonObject()) {
JsonObject jsonObject = aJsonElement.getAsJsonObject();
if (jsonObject.has(JSON_OBJECT_MEDIAARRAY)) {
Map<Qualities, UrlWidthHeightDTO> urlMap = new HashMap<>();
Map<Qualities, String> urlMap = new HashMap<>();

JsonArray mediaArray = jsonObject.get(JSON_OBJECT_MEDIAARRAY).getAsJsonArray();
mediaArray.forEach(mediaArrayItem -> {
JsonObject item = mediaArrayItem.getAsJsonObject();
JsonArray streamArray = item.get(JSON_OBJECT_MEDIASTREAMARRAY).getAsJsonArray();
streamArray.forEach(streamItem -> {
deserializeMediaStreamArrayItem(streamItem.getAsJsonObject(), urlMap);
});
});
if (mediaArray.size() > 0) {
JsonElement mediaArrayItem = mediaArray.get(mediaArray.size() - 1);
JsonObject item = mediaArrayItem.getAsJsonObject();
JsonArray streamArray = item.get(JSON_OBJECT_MEDIASTREAMARRAY).getAsJsonArray();
deserializeMediaStreamArray(streamArray, urlMap);
}

// URLs setzen
urlMap.forEach((key, value) -> {
dto.addVideo(key, value.url);
dto.addVideo(key, value);
});
}
}

return dto;
}

private void deserializeMediaStreamArrayItem(JsonObject stream, Map<Qualities, UrlWidthHeightDTO> urlMap) {
private void deserializeMediaStreamArray(JsonArray streamArray, Map<Qualities, String> urlMap) {
List<QualityUrlWidthHeightDTO> qualities = new ArrayList<>();

streamArray.forEach(streamItem -> {
qualities.add(deserializeUrlInfos(streamItem.getAsJsonObject()));
});

convertQualities(qualities, urlMap);
}

private void convertQualities(List<QualityUrlWidthHeightDTO> qualities, Map<Qualities, String> urlMap) {
// bei der besten Qualität anfangen
qualities.sort((a, b) -> b.quality.compareTo(a.quality));

for (int i = 0; i < qualities.size(); i++) {
QualityUrlWidthHeightDTO qualityDto = qualities.get(i);
if (qualityDto.urls.isEmpty()) {
continue;
}

String qualityValue = stream.get(JSON_ELEMENT_QUALITY).getAsString();

UrlWidthHeightDTO urlInfos = deserializeUrlInfos(stream);

Qualities quality = convertQuality(qualityValue, urlInfos);
// nur relevante Qualitäten berücksichtigen und nur http-Links (manchmal sind "mp3:"-Links enthalten)
if (quality != null && urlInfos.url.startsWith(TEXT_START_HTTP)) {
if (urlMap.containsKey(quality)) {
UrlWidthHeightDTO actualUrlInfos = urlMap.get(quality);

// prüfen, ob die Auflösung besser ist, als die bisher für die Qualität hinterlegte
if (urlInfos.width > 0 && urlInfos.width > actualUrlInfos.width) {
urlMap.put(quality, urlInfos);
} else if (quality == Qualities.NORMAL) {
// bei normaler Auflösung ist aus irgendeinem Grund der letzte
// Eintrag immer der beste!
urlMap.put(quality, urlInfos);
}
} else {
urlMap.put(quality, urlInfos);
switch(qualityDto.quality){
case "1":
// für SMALL die erste URL nehmen, ist immer die bessere
addUrl(urlMap, Qualities.SMALL, qualityDto.urls.get(0));
break;
case "2":
// für NORMAL die letzte URL nehmen, ist immer die bessere
addUrl(urlMap, Qualities.NORMAL, qualityDto.urls.get(qualityDto.urls.size()-1));
break;
case "3":
if (qualityDto.width >= 1280 && qualityDto.height >= 720) {
addUrl(urlMap, Qualities.HD, getUrl(qualityDto, qualities.get(i+1)));
}
}
if (qualityDto.width == 0 && qualityDto.height == 0) {
String url = getUrl(qualityDto, qualities.get(i+1));

// Die Prüfung auf den Dateinamen der URL filtert noch zusätzlich
// Filme heraus, die eine Auflösung von 960x... haben
if (!url.substring(url.lastIndexOf('/') + 1).startsWith("960")) {
addUrl(urlMap, Qualities.HD, url);
} else {
addUrl(urlMap, Qualities.NORMAL, url);
}
}
break;
}
}
}

private UrlWidthHeightDTO deserializeUrlInfos(JsonObject stream) {
UrlWidthHeightDTO dto = new UrlWidthHeightDTO();
private void addUrl(Map<Qualities, String> urlMap, Qualities quality, String url) {
if (url != null && !url.isEmpty() && !urlMap.containsKey(quality)) {
urlMap.put(quality, url);
}
}

private String getUrl(QualityUrlWidthHeightDTO quality, QualityUrlWidthHeightDTO lowerQuality) {
for (int i = 0; i < quality.urls.size(); i++) {
String url = quality.urls.get(i);

if (!lowerQuality.urls.contains(url)) {
return url;
}
}

return "";
}

private QualityUrlWidthHeightDTO deserializeUrlInfos(JsonObject stream) {
QualityUrlWidthHeightDTO dto = new QualityUrlWidthHeightDTO();

dto.quality = stream.get(JSON_ELEMENT_QUALITY).getAsString();

// Url ermitteln
// Wenn es ein Array ist, den letzten Eintrag nehmen, der hat im Zweifel
// die bessere Auflösung!
JsonElement streamElement = stream.get(JSON_ELEMENT_STREAM);
if(streamElement.isJsonArray()) {
JsonArray streamArray = streamElement.getAsJsonArray();
dto.url = streamArray.get(streamArray.size()-1).getAsString();
streamArray.forEach(arrayElement -> {
dto.urls.add(addMissingHttpPrefixIfNecessary(arrayElement.getAsString()));
});
} else {
dto.url = streamElement.getAsString();
dto.urls.add(addMissingHttpPrefixIfNecessary(streamElement.getAsString()));
}
dto.url = addMissingHttpPrefixIfNecessary(dto.url);

// Auflösung ermitteln
if( stream.has(JSON_ELEMENT_WIDTH)) {
Expand All @@ -101,37 +151,11 @@ private UrlWidthHeightDTO deserializeUrlInfos(JsonObject stream) {
return dto;
}

private static String addMissingHttpPrefixIfNecessary(String aUrl) {
if(aUrl.startsWith("//")) {
aUrl = TEXT_START_HTTP + ":" + aUrl;
}

return aUrl;
}

private static Qualities convertQuality(String quality, UrlWidthHeightDTO urlInfos) {
switch(quality) {
case "1":
return Qualities.SMALL;
case "2":
return Qualities.NORMAL;
case "3":
// Beste Qualität, aber nicht immer ist die Auflösung auch HD
// Die Prüfung auf den Dateinamen der URL filtert noch zusätzlich
// Filme heraus, die eine Auflösung von 960x... haben
if ((urlInfos.width == 0 && urlInfos.height == 0 && !urlInfos.url.substring(urlInfos.url.lastIndexOf("/") + 1).startsWith("960"))
|| (urlInfos.width >= 1280 && urlInfos.height >= 720)) {
return Qualities.HD;
} else {
return Qualities.NORMAL;
}
}
return null;
}

// Hilfsklasse für Zwischenspeichern von URL und zugehörigen Auflösungsinfos
private class UrlWidthHeightDTO {
public String url = "";
private class QualityUrlWidthHeightDTO {
public String quality = "";
public List<String> urls = new ArrayList<>();
public int width;
public int height;
}
Expand Down
9 changes: 7 additions & 2 deletions src/test/developTest/java/mServer/crawler/sender/ard/ArdVideoDeserializerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
import mServer.test.JsonFileReader;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.notNullValue;
import org.junit.Test;
import static org.junit.Assert.*;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

Expand All @@ -22,7 +22,12 @@ public static Collection<Object[]> data() {
{ "/ard/ard_video_with_hd.json", "http://ondemand.mdr.de/mp4dyn/1/FCMS-14ec1b13-c7f6-4cc4-8b72-681cd22da39a-e9ebd6e42ce1_14.mp4", "http://ondemand.mdr.de/mp4dyn/1/FCMS-14ec1b13-c7f6-4cc4-8b72-681cd22da39a-c7cca1d51b4b_14.mp4", "http://ondemand.mdr.de/mp4dyn/1/FCMS-14ec1b13-c7f6-4cc4-8b72-681cd22da39a-15e7604ea4a4_14.mp4" },
{ "/ard/ard_video_normal_use_last.json", "https://mediastorage01.sr-online.de/Video/UD/DOKU/1505155201_20170911_KANDIDATENCHECK_LUKSIC_M.mp4", "https://srstorage01-a.akamaihd.net/Video/UD/DOKU/1505155201_20170911_KANDIDATENCHECK_LUKSIC_L.mp4", "https://srstorage01-a.akamaihd.net/Video/UD/DOKU/1505155201_20170911_KANDIDATENCHECK_LUKSIC_P.mp4" },
{ "/ard/ard_video_use_http_url.json", "http://cdn-storage.br.de/iLCpbHJGNL9zu6i6NL97bmWH_-bf/_-0S/_Abg5-xg5U1S/0f131ba9-c8e1-4368-be7b-799a75df221f_2.mp3", null, null },
{ "/ard/ard_video_with_quality_3_no_hd.json", "http://pd-videos.daserste.de/int/2017/09/08/7f35d2d7-9854-4187-b406-c56b9292de79/512-1.mp4", "http://pd-videos.daserste.de/int/2017/09/08/7f35d2d7-9854-4187-b406-c56b9292de79/960-1.mp4", null }
{ "/ard/ard_video_with_quality_3_no_hd.json", "http://pd-videos.daserste.de/int/2017/09/08/7f35d2d7-9854-4187-b406-c56b9292de79/512-1.mp4", "http://pd-videos.daserste.de/int/2017/09/08/7f35d2d7-9854-4187-b406-c56b9292de79/960-1.mp4", null },
{ "/ard/ard_video_ndr_with_hd.json", "https://mediandr-a.akamaihd.net/progressive/2017/0915/TV-20170915-1645-5500.hi.mp4", "https://mediandr-a.akamaihd.net/progressive/2017/0915/TV-20170915-1645-5500.hq.mp4", "https://mediandr-a.akamaihd.net/progressive/2017/0915/TV-20170915-1645-5500.hd.mp4" },
{ "/ard/ard_video_swr_with_hd.json", "https://pdodswr-a.akamaihd.net/swr/swr-fernsehen/lust-auf-backen/985691.m.mp4", "https://pdodswr-a.akamaihd.net/swr/swr-fernsehen/lust-auf-backen/985691.l.mp4", "https://pdodswr-a.akamaihd.net/swr/swr-fernsehen/lust-auf-backen/985691.xl.mp4" },
{ "/ard/ard_video_ard_with_hd.json", "https://media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.webm.h264.mp4", "https://media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.webl.h264.mp4", "https://media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.webxl.h264.mp4" },
{ "/ard/ard_video_hr_with_hd.json", "http://hrardmediathek-a.akamaihd.net/video/as/allewetter/2017_12/hrLogo_171228193505_L279621_512x288-25p-500kbit.mp4", "http://hrardmediathek-a.akamaihd.net/video/as/allewetter/2017_12/hrLogo_171228193505_L279621_960x540-50p-1800kbit.mp4", "http://hrardmediathek-a.akamaihd.net/video/as/allewetter/2017_12/hrLogo_171228193505_L279621_1280x720-50p-5000kbit.mp4" },
{ "/ard/ard_video_mdr_with_hd.json", "https://odgeomdr-a.akamaihd.net/mp4dyn/7/FCMS-74bf126c-63dc-490d-a256-6c90aa6a21a6-e9ebd6e42ce1_74.mp4", "https://odgeomdr-a.akamaihd.net/mp4dyn/7/FCMS-74bf126c-63dc-490d-a256-6c90aa6a21a6-c7cca1d51b4b_74.mp4", "https://odgeomdr-a.akamaihd.net/mp4dyn/7/FCMS-74bf126c-63dc-490d-a256-6c90aa6a21a6-15e7604ea4a4_74.mp4" },
});
}

Expand Down
91 changes: 91 additions & 0 deletions src/test/developTest/resources/ard/ard_video_ard_with_hd.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
{
"_type":"video",
"_isLive":false,
"_defaultQuality":[
"auto",
2,
4,
3,
1,
0
],
"_previewImage":"https://img.ardmediathek.de/standard/00/48/77/78/44/-1774185891/16x9/960?mandant=ard",
"_subtitleUrl":"http://www.ardmediathek.de/subtitle/210382",
"_subtitleOffset":0,
"_mediaArray":[
{
"_plugin":0,
"_mediaStreamArray":[
{
"_quality":"auto",
"_server":"",
"_cdn":"akamai",
"_stream":"http://hlstagesschau-vh.akamaihd.net/z/video/2017/1229/TV-20171229-1006-5301.,webm,webml,webl,webxl,.h264.mp4.csmil/manifest.f4m"
},
{
"_quality":1,
"_server":"",
"_cdn":"default",
"_stream":"https://download.media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.websm.h264.mp4"
},
{
"_quality":2,
"_server":"",
"_cdn":"default",
"_width":640,
"_height":360,
"_stream":"https://media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.webml.h264.mp4"
}
]
},
{
"_plugin":1,
"_mediaStreamArray":[
{
"_quality":"auto",
"_server":"",
"_cdn":"flashls",
"_stream":"https://hlstagesschau-vh.akamaihd.net/i/video/2017/1229/TV-20171229-1006-5301.,webs,websm,webm,webml,webl,webxl,.h264.mp4.csmil/master.m3u8"
},
{
"_quality":0,
"_server":"",
"_cdn":"default",
"_stream":"https://media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.webs.h264.mp4"
},
{
"_quality":1,
"_stream":[
"https://media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.webm.h264.mp4",
"https://media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.websm.h264.mp4",
"https://download.media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.websm.h264.mp4"
]
},
{
"_quality":2,
"_stream":[
"https://media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.webml.h264.mp4",
"https://media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.webl.h264.mp4"
]
},
{
"_quality":3,
"_stream":[
"https://media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.webxl.h264.mp4",
"https://media.tagesschau.de/video/2017/1229/TV-20171229-1006-5301.webl.h264.mp4"
]
}
]
}
],
"_alternativeMediaArray":[

],
"_sortierArray":[
1,
0
],
"_duration":306,
"_dvrEnabled":false,
"_geoblocked":false
}
Loading

0 comments on commit 598db97

Please sign in to comment.