diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py index 97ca684ff56..acfa109f2fc 100755 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -774,8 +774,6 @@ def testSolrExample(binaryDistPath, javaPath): raise RuntimeError('Failed to run the techproducts example, check log for previous errors.') os.chdir('example') - print(' test utf8...') - run('sh ./exampledocs/test_utf8.sh http://localhost:8983/solr/techproducts', 'utf8.log') print(' run query...') s = load('http://localhost:8983/solr/techproducts/select/?q=video') if s.find('"numFound":3,') == -1: diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 1b60d044ad0..1eff932d698 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -73,6 +73,8 @@ Deprecation Removals * SOLR-17400: Remove deprecated script snapshotcli.sh. bin/solr snapshot-* commands have replaced this. (Eric Pugh) +* SOLR-17494: Remove language specific writer types (i.e wt= ruby, python, php, and phps). (Eric Pugh) + Dependency Upgrades --------------------- (No changes) diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java index 15b4526d49e..3c3217cc085 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java @@ -59,17 +59,7 @@ public class QueryResponseWriters { public static class BenchState { /** See {@link SolrCore#DEFAULT_RESPONSE_WRITERS} */ - @Param({ - CommonParams.JAVABIN, - CommonParams.JSON, - "cbor", - "smile", - "xml", - "python", - "phps", - "ruby", - "raw" - }) + @Param({CommonParams.JAVABIN, CommonParams.JSON, "cbor", "smile", "xml", "raw"}) String wt; private int docs = 100; diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index da78f08fdc4..f56cb846767 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -128,13 +128,9 @@ import org.apache.solr.response.GeoJSONResponseWriter; import org.apache.solr.response.GraphMLResponseWriter; import org.apache.solr.response.JacksonJsonWriter; -import org.apache.solr.response.PHPResponseWriter; -import org.apache.solr.response.PHPSerializedResponseWriter; import org.apache.solr.response.PrometheusResponseWriter; -import org.apache.solr.response.PythonResponseWriter; import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.RawResponseWriter; -import org.apache.solr.response.RubyResponseWriter; import org.apache.solr.response.SchemaXmlResponseWriter; import org.apache.solr.response.SmileResponseWriter; import org.apache.solr.response.SolrQueryResponse; @@ -3015,10 +3011,6 @@ public PluginBag getResponseWriters() { m.put("standard", m.get(CommonParams.JSON)); m.put("geojson", new GeoJSONResponseWriter()); m.put("graphml", new GraphMLResponseWriter()); - m.put("python", new PythonResponseWriter()); - m.put("php", new PHPResponseWriter()); - m.put("phps", new PHPSerializedResponseWriter()); - m.put("ruby", new RubyResponseWriter()); m.put("raw", new RawResponseWriter()); m.put(CommonParams.JAVABIN, new BinaryResponseWriter()); m.put("cbor", new CborResponseWriter()); diff --git a/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java deleted file mode 100644 index b9f4c3417fd..00000000000 --- a/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.response; - -import java.io.IOException; -import java.io.Writer; -import java.util.List; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.request.SolrQueryRequest; - -public class PHPResponseWriter implements QueryResponseWriter { - static String CONTENT_TYPE_PHP_UTF8 = "text/x-php;charset=UTF-8"; - - private String contentType = CONTENT_TYPE_PHP_UTF8; - - @Override - public void init(NamedList namedList) { - String contentType = (String) namedList.get("content-type"); - if (contentType != null) { - this.contentType = contentType; - } - } - - @Override - public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - PHPWriter w = new PHPWriter(writer, req, rsp); - try { - w.writeResponse(); - } finally { - w.close(); - } - } - - @Override - public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { - return contentType; - } -} - -class PHPWriter extends JSONWriter { - public PHPWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) { - super(writer, req, rsp); - } - - @Override - public void writeNamedList(String name, NamedList val) throws IOException { - writeNamedListAsMapMangled(name, val); - } - - @Override - public void writeMapOpener(int size) throws IOException { - writer.write("array("); - } - - @Override - public void writeMapCloser() throws IOException { - writer.write(')'); - } - - @Override - public void writeArrayOpener(int size) throws IOException { - writer.write("array("); - } - - @Override - public void writeArray(String name, List l, boolean raw) throws IOException { - writeArray(name, l.iterator(), raw); - } - - @Override - public void writeArrayCloser() throws IOException { - writer.write(')'); - } - - @Override - public void writeNull(String name) throws IOException { - writer.write("null"); - } - - @Override - public void writeKey(String fname, boolean needsEscaping) throws IOException { - writeStr(null, fname, needsEscaping); - writer.write('='); - writer.write('>'); - } - - @Override - public void writeStr(String name, String val, boolean needsEscaping) throws IOException { - if (needsEscaping) { - writer.write('\''); - for (int i = 0; i < val.length(); i++) { - char ch = val.charAt(i); - switch (ch) { - case '\'': - case '\\': - writer.write('\\'); - writer.write(ch); - break; - default: - writer.write(ch); - } - } - writer.write('\''); - } else { - writer.write('\''); - writer.write(val); - writer.write('\''); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java deleted file mode 100644 index 0a6f828936f..00000000000 --- a/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.response; - -import java.io.IOException; -import java.io.Writer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.Map; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.UnicodeUtil; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.search.ReturnFields; - -/** - * A description of the PHP serialization format can be found here: - * http://www.hurring.com/scott/code/perl/serialize/ - */ -public class PHPSerializedResponseWriter implements QueryResponseWriter { - static String CONTENT_TYPE_PHP_UTF8 = "text/x-php-serialized;charset=UTF-8"; - - private String contentType = CONTENT_TYPE_PHP_UTF8; - - @Override - public void init(NamedList namedList) { - String contentType = (String) namedList.get("content-type"); - if (contentType != null) { - this.contentType = contentType; - } - } - - @Override - public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - PHPSerializedWriter w = new PHPSerializedWriter(writer, req, rsp); - try { - w.writeResponse(); - } finally { - w.close(); - } - } - - @Override - public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { - return contentType; - } -} - -class PHPSerializedWriter extends JSONWriter { - byte[] utf8; - - public PHPSerializedWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) { - super(writer, req, rsp); - this.utf8 = BytesRef.EMPTY_BYTES; - // never indent serialized PHP data - doIndent = false; - } - - @Override - public void writeResponse() throws IOException { - Boolean omitHeader = req.getParams().getBool(CommonParams.OMIT_HEADER); - if (omitHeader != null && omitHeader) rsp.removeResponseHeader(); - writeNamedList(null, rsp.getValues()); - } - - @Override - public void writeNamedList(String name, NamedList val) throws IOException { - writeNamedListAsMapMangled(name, val); - } - - @Override - public void writeStartDocumentList( - String name, long start, int size, long numFound, Float maxScore, Boolean numFoundExact) - throws IOException { - writeMapOpener(headerSize(maxScore, numFoundExact)); - writeKey("numFound", false); - writeLong(null, numFound); - writeKey("start", false); - writeLong(null, start); - - if (maxScore != null) { - writeKey("maxScore", false); - writeFloat(null, maxScore); - } - if (numFoundExact != null) { - writeKey("numFoundExact", false); - writeBool(null, numFoundExact); - } - writeKey("docs", false); - writeArrayOpener(size); - } - - @Override - public void writeEndDocumentList() throws IOException { - writeArrayCloser(); // doc list - writeMapCloser(); - } - - @Override - public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx) - throws IOException { - writeKey(idx, false); - - LinkedHashMap single = new LinkedHashMap<>(); - LinkedHashMap multi = new LinkedHashMap<>(); - - for (String fname : doc.getFieldNames()) { - if (returnFields != null && !returnFields.wantsField(fname)) { - continue; - } - - Object val = doc.getFieldValue(fname); - if (val instanceof Collection) { - multi.put(fname, val); - } else { - single.put(fname, val); - } - } - - writeMapOpener(single.size() + multi.size()); - for (Map.Entry entry : single.entrySet()) { - String fname = entry.getKey(); - Object val = entry.getValue(); - writeKey(fname, true); - writeVal(fname, val); - } - - for (Map.Entry entry : multi.entrySet()) { - String fname = entry.getKey(); - writeKey(fname, true); - - Object val = entry.getValue(); - if (!(val instanceof Collection)) { - // should never be reached if multivalued fields are stored as a Collection - // so I'm assuming a size of 1 just to wrap the single value - writeArrayOpener(1); - writeVal(fname, val); - writeArrayCloser(); - } else { - writeVal(fname, val); - } - } - - writeMapCloser(); - } - - @Override - public void writeArray(String name, Object[] val, boolean raw) throws IOException { - assert !raw; - writeMapOpener(val.length); - for (int i = 0; i < val.length; i++) { - writeKey(i, false); - writeVal(String.valueOf(i), val[i]); - } - writeMapCloser(); - } - - @Override - public void writeArray(String name, Iterator val, boolean raw) throws IOException { - assert !raw; - ArrayList vals = new ArrayList<>(); - while (val.hasNext()) { - vals.add(val.next()); - } - writeArray(name, vals.toArray(), false); - } - - @Override - public void writeMapOpener(int size) throws IOException, IllegalArgumentException { - // negative size value indicates that something has gone wrong - if (size < 0) { - throw new IllegalArgumentException("Map size must not be negative"); - } - writer.write("a:" + size + ":{"); - } - - @Override - public void writeMapSeparator() throws IOException { - /* NOOP */ - } - - @Override - public void writeMapCloser() throws IOException { - writer.write('}'); - } - - @Override - public void writeArrayOpener(int size) throws IOException, IllegalArgumentException { - // negative size value indicates that something has gone wrong - if (size < 0) { - throw new IllegalArgumentException("Array size must not be negative"); - } - writer.write("a:" + size + ":{"); - } - - @Override - public void writeArraySeparator() throws IOException { - /* NOOP */ - } - - @Override - public void writeArrayCloser() throws IOException { - writer.write('}'); - } - - @Override - public void writeNull(String name) throws IOException { - writer.write("N;"); - } - - @Override - public void writeKey(String fname, boolean needsEscaping) throws IOException { - writeStr(null, fname, needsEscaping); - } - - void writeKey(int val, boolean needsEscaping) throws IOException { - writeInt(null, String.valueOf(val)); - } - - @Override - public void writeBool(String name, boolean val) throws IOException { - writer.write(val ? "b:1;" : "b:0;"); - } - - @Override - public void writeBool(String name, String val) throws IOException { - writeBool(name, val.charAt(0) == 't'); - } - - @Override - public void writeInt(String name, String val) throws IOException { - writer.write("i:" + val + ";"); - } - - @Override - public void writeLong(String name, String val) throws IOException { - writeInt(name, val); - } - - @Override - public void writeFloat(String name, String val) throws IOException { - writeDouble(name, val); - } - - @Override - public void writeDouble(String name, String val) throws IOException { - writer.write("d:" + val + ";"); - } - - @Override - public void writeStr(String name, String val, boolean needsEscaping) throws IOException { - // serialized PHP strings don't need to be escaped at all, however the - // string size reported needs be the number of bytes rather than chars. - utf8 = ArrayUtil.grow(utf8, val.length() * UnicodeUtil.MAX_UTF8_BYTES_PER_CHAR); - final int nBytes = UnicodeUtil.UTF16toUTF8(val, 0, val.length(), utf8); - - writer.write("s:"); - writer.write(Integer.toString(nBytes)); - writer.write(":\""); - writer.write(val); - writer.write("\";"); - } -} diff --git a/solr/core/src/java/org/apache/solr/response/PythonResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PythonResponseWriter.java deleted file mode 100644 index d50fb95e287..00000000000 --- a/solr/core/src/java/org/apache/solr/response/PythonResponseWriter.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.response; - -import java.io.IOException; -import java.io.Writer; -import org.apache.solr.request.SolrQueryRequest; - -public class PythonResponseWriter implements QueryResponseWriter { - static String CONTENT_TYPE_PYTHON_ASCII = "text/x-python;charset=US-ASCII"; - - @Override - public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - PythonWriter w = new PythonWriter(writer, req, rsp); - try { - w.writeResponse(); - } finally { - w.close(); - } - } - - @Override - public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { - return CONTENT_TYPE_TEXT_ASCII; - } -} - -class PythonWriter extends JSONResponseWriter.NaNFloatWriter { - @Override - protected String getNaN() { - return "float('NaN')"; - } - - @Override - protected String getInf() { - return "float('Inf')"; - } - - public PythonWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) { - super(writer, req, rsp); - } - - @Override - public void writeNull(String name) throws IOException { - writer.write("None"); - } - - @Override - public void writeBool(String name, boolean val) throws IOException { - writer.write(val ? "True" : "False"); - } - - @Override - public void writeBool(String name, String val) throws IOException { - writeBool(name, val.charAt(0) == 't'); - } - - /* optionally use a unicode python string if necessary */ - @Override - public void writeStr(String name, String val, boolean needsEscaping) throws IOException { - if (!needsEscaping) { - writer.write('\''); - writer.write(val); - writer.write('\''); - return; - } - - // use python unicode strings... - // python doesn't tolerate newlines in strings in its eval(), so we must escape them. - - StringBuilder sb = new StringBuilder(val.length()); - boolean needUnicode = false; - - for (int i = 0; i < val.length(); i++) { - char ch = val.charAt(i); - switch (ch) { - case '\'': - case '\\': - sb.append('\\'); - sb.append(ch); - break; - case '\r': - sb.append("\\r"); - break; - case '\n': - sb.append("\\n"); - break; - case '\t': - sb.append("\\t"); - break; - default: - // we don't strictly have to escape these chars, but it will probably increase - // portability to stick to visible ascii - if (ch < ' ' || ch > 127) { - unicodeEscape(sb, ch); - needUnicode = true; - } else { - sb.append(ch); - } - } - } - - if (needUnicode) { - writer.write('u'); - } - writer.write('\''); - writer.append(sb); - writer.write('\''); - } - - /* - old version that always used unicode - public void writeStr(String name, String val, boolean needsEscaping) throws IOException { - // use python unicode strings... - // python doesn't tolerate newlines in strings in its eval(), so we must escape them. - writer.write("u'"); - // it might be more efficient to use a stringbuilder or write substrings - // if writing chars to the stream is slow. - if (needsEscaping) { - for (int i=0; i127) { - unicodeChar(ch); - } else { - writer.write(ch); - } - } - } - } else { - writer.write(val); - } - writer.write('\''); - } - */ - -} diff --git a/solr/core/src/java/org/apache/solr/response/RubyResponseWriter.java b/solr/core/src/java/org/apache/solr/response/RubyResponseWriter.java deleted file mode 100644 index bce2c2c9498..00000000000 --- a/solr/core/src/java/org/apache/solr/response/RubyResponseWriter.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.response; - -import java.io.IOException; -import java.io.Writer; -import org.apache.solr.request.SolrQueryRequest; - -public class RubyResponseWriter implements QueryResponseWriter { - static String CONTENT_TYPE_RUBY_UTF8 = "text/x-ruby;charset=UTF-8"; - - @Override - public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - RubyWriter w = new RubyWriter(writer, req, rsp); - try { - w.writeResponse(); - } finally { - w.close(); - } - } - - @Override - public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { - return CONTENT_TYPE_TEXT_UTF8; - } -} - -class RubyWriter extends JSONResponseWriter.NaNFloatWriter { - - @Override - protected String getNaN() { - return "(0.0/0.0)"; - } - - @Override - protected String getInf() { - return "(1.0/0.0)"; - } - - public RubyWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) { - super(writer, req, rsp); - } - - @Override - public void writeNull(String name) throws IOException { - writer.write("nil"); - } - - @Override - public void writeKey(String fname, boolean needsEscaping) throws IOException { - writeStr(null, fname, needsEscaping); - writer.write('='); - writer.write('>'); - } - - @Override - public void writeStr(String name, String val, boolean needsEscaping) throws IOException { - // Ruby doesn't do unicode escapes... so let the servlet container write raw UTF-8 - // bytes into the string. - // - // Use single quoted strings for safety since no evaluation is done within them. - // Also, there are very few escapes recognized in a single quoted string, so - // only escape the backslash and single quote. - writer.write('\''); - if (needsEscaping) { - for (int i = 0; i < val.length(); i++) { - char ch = val.charAt(i); - if (ch == '\'' || ch == '\\') { - writer.write('\\'); - } - writer.write(ch); - } - } else { - writer.write(val); - } - writer.write('\''); - } -} diff --git a/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java b/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java index 6d45634da78..5ccf19cdecf 100644 --- a/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java +++ b/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java @@ -39,7 +39,7 @@ import org.junit.BeforeClass; import org.junit.Test; -/** Test some aspects of JSON/python writer output (very incomplete) */ +/** Test some aspects of JSON writer output */ public class JSONWriterTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { @@ -53,25 +53,17 @@ private void jsonEq(String expected, String received) { } @Test - public void testTypes() throws IOException { + public void testSimpleJson() throws IOException { SolrQueryRequest req = req("q", "dummy", "indent", "off"); SolrQueryResponse rsp = new SolrQueryResponse(); - QueryResponseWriter w = new PythonResponseWriter(); + QueryResponseWriter w = new JSONResponseWriter(); StringWriter buf = new StringWriter(); + rsp.add("data1", Float.NaN); rsp.add("data2", Double.NEGATIVE_INFINITY); rsp.add("data3", Float.POSITIVE_INFINITY); - w.write(buf, req, rsp); - jsonEq(buf.toString(), "{'data1':float('NaN'),'data2':-float('Inf'),'data3':float('Inf')}"); - - w = new RubyResponseWriter(); - buf = new StringWriter(); - w.write(buf, req, rsp); - jsonEq(buf.toString(), "{'data1'=>(0.0/0.0),'data2'=>-(1.0/0.0),'data3'=>(1.0/0.0)}"); - w = new JSONResponseWriter(); - buf = new StringWriter(); w.write(buf, req, rsp); jsonEq(buf.toString(), "{\"data1\":\"NaN\",\"data2\":\"-Infinity\",\"data3\":\"Infinity\"}"); req.close(); @@ -326,7 +318,7 @@ public void testWfrJacksonJsonWriter() throws IOException { JacksonJsonWriter w = new JacksonJsonWriter(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); w.write(baos, req, rsp); - String received = new String(baos.toByteArray(), StandardCharsets.UTF_8); + String received = baos.toString(StandardCharsets.UTF_8); String expected = "testFun( {\n \"param0\":\"v0\",\n \"param1\":42\n} )"; jsonEq(expected, received); req.close(); diff --git a/solr/core/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java deleted file mode 100644 index de6339650a3..00000000000 --- a/solr/core/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.response; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.Arrays; -import java.util.LinkedHashMap; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.request.SolrQueryRequest; -import org.junit.BeforeClass; -import org.junit.Test; - -/** Basic PHPS tests based on JSONWriterTest */ -public class TestPHPSerializedResponseWriter extends SolrTestCaseJ4 { - @BeforeClass - public static void beforeClass() throws Exception { - initCore("solrconfig.xml", "schema.xml"); - } - - @Test - public void testSimple() throws IOException { - SolrQueryRequest req = req("dummy"); - SolrQueryResponse rsp = new SolrQueryResponse(); - QueryResponseWriter w = new PHPSerializedResponseWriter(); - - StringWriter buf = new StringWriter(); - rsp.add("data1", "hello"); - rsp.add("data2", 42); - rsp.add("data3", true); - w.write(buf, req, rsp); - assertEquals( - "a:3:{s:5:\"data1\";s:5:\"hello\";s:5:\"data2\";i:42;s:5:\"data3\";b:1;}", buf.toString()); - req.close(); - } - - @Test - public void testSolrDocuments() throws IOException { - SolrQueryRequest req = req("q", "*:*"); - SolrQueryResponse rsp = new SolrQueryResponse(); - QueryResponseWriter w = new PHPSerializedResponseWriter(); - StringWriter buf = new StringWriter(); - - SolrDocument d = new SolrDocument(); - - SolrDocument d1 = d; - d.addField("id", "1"); - d.addField("data1", "hello"); - d.addField("data2", 42); - d.addField("data3", true); - - // multivalued fields: - - // extremely odd edge case: value is a map - - // we use LinkedHashMap because we are doing a string comparison - // later, and we need predictable ordering - LinkedHashMap nl = new LinkedHashMap<>(); - nl.put("data4.1", "hashmap"); - nl.put("data4.2", "hello"); - d.addField("data4", nl); - // array value - d.addField("data5", Arrays.asList("data5.1", "data5.2", "data5.3")); - - // adding one more document to test array indexes - d = new SolrDocument(); - SolrDocument d2 = d; - d.addField("id", "2"); - - SolrDocumentList sdl = new SolrDocumentList(); - sdl.add(d1); - sdl.add(d2); - rsp.addResponse(sdl); - - w.write(buf, req, rsp); - assertEquals( - "a:1:{s:8:\"response\";a:4:{s:8:\"numFound\";i:0;s:5:\"start\";i:0;s:13:\"numFoundExact\";b:1;s:4:\"docs\";a:2:{i:0;a:6:{s:2:\"id\";s:1:\"1\";s:5:\"data1\";s:5:\"hello\";s:5:\"data2\";i:42;s:5:\"data3\";b:1;s:5:\"data4\";a:2:{s:7:\"data4.1\";s:7:\"hashmap\";s:7:\"data4.2\";s:5:\"hello\";}s:5:\"data5\";a:3:{i:0;s:7:\"data5.1\";i:1;s:7:\"data5.2\";i:2;s:7:\"data5.3\";}}i:1;a:1:{s:2:\"id\";s:1:\"2\";}}}}", - buf.toString()); - req.close(); - } -} diff --git a/solr/example/exampledocs/test_utf8.sh b/solr/example/exampledocs/test_utf8.sh deleted file mode 100755 index 9032e12ffe3..00000000000 --- a/solr/example/exampledocs/test_utf8.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/sh -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#Test script to tell if the server is accepting UTF-8 -#The python writer currently escapes non-ascii chars, so it's good for testing - -SOLR_URL=http://localhost:8983/solr - -if [ ! -z $1 ]; then - SOLR_URL=$1 -fi - -curl "$SOLR_URL/select?q=hello¶ms=explicit&wt=python" 2> /dev/null | grep 'hello' > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "Solr server is up." -else - echo "ERROR: Could not curl to Solr - is curl installed? Is Solr not running?" - exit 1 -fi - -curl "$SOLR_URL/select?q=h%C3%A9llo&echoParams=explicit&wt=python" 2> /dev/null | grep 'h\\u00e9llo' > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP GET is accepting UTF-8" -else - echo "ERROR: HTTP GET is not accepting UTF-8" -fi - -curl $SOLR_URL/select --data-binary 'q=h%C3%A9llo&echoParams=explicit&wt=python' -H 'Content-type:application/x-www-form-urlencoded; charset=UTF-8' 2> /dev/null | grep 'h\\u00e9llo' > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP POST is accepting UTF-8" -else - echo "ERROR: HTTP POST is not accepting UTF-8" -fi - -curl $SOLR_URL/select --data-binary 'q=h%C3%A9llo&echoParams=explicit&wt=python' 2> /dev/null | grep 'h\\u00e9llo' > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP POST defaults to UTF-8" -else - echo "HTTP POST does not default to UTF-8" -fi - - -#A unicode character outside of the BMP (a circle with an x inside) -CHAR="𐌈" -CODEPOINT='0x10308' -#URL encoded UTF8 of the codepoint -UTF8_Q='%F0%90%8C%88' -#expected return of the python writer (currently uses UTF-16 surrogates) -EXPECTED='\\ud800\\udf08' - -curl "$SOLR_URL/select?q=$UTF8_Q&echoParams=explicit&wt=python" 2> /dev/null | grep $EXPECTED > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP GET is accepting UTF-8 beyond the basic multilingual plane" -else - echo "ERROR: HTTP GET is not accepting UTF-8 beyond the basic multilingual plane" -fi - -curl $SOLR_URL/select --data-binary "q=$UTF8_Q&echoParams=explicit&wt=python" -H 'Content-type:application/x-www-form-urlencoded; charset=UTF-8' 2> /dev/null | grep $EXPECTED > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP POST is accepting UTF-8 beyond the basic multilingual plane" -else - echo "ERROR: HTTP POST is not accepting UTF-8 beyond the basic multilingual plane" -fi - -curl "$SOLR_URL/select?q=$UTF8_Q&echoParams=explicit&wt=python" --data-binary '' 2> /dev/null | grep $EXPECTED > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP POST + URL params is accepting UTF-8 beyond the basic multilingual plane" -else - echo "ERROR: HTTP POST + URL params is not accepting UTF-8 beyond the basic multilingual plane" -fi - -#curl "$SOLR_URL/select?q=$UTF8_Q&echoParams=explicit" 2> /dev/null | od -tx1 -w1000 | sed 's/ //g' | grep 'f4808198' > /dev/null 2>&1 -curl "$SOLR_URL/select?q=$UTF8_Q&echoParams=explicit" 2> /dev/null | grep "$CHAR" > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "Response correctly returns UTF-8 beyond the basic multilingual plane" -else - echo "ERROR: Response can't return UTF-8 beyond the basic multilingual plane" -fi - - diff --git a/solr/server/solr/configsets/_default/conf/solrconfig.xml b/solr/server/solr/configsets/_default/conf/solrconfig.xml index 825993191ae..5bc2cf13415 100644 --- a/solr/server/solr/configsets/_default/conf/solrconfig.xml +++ b/solr/server/solr/configsets/_default/conf/solrconfig.xml @@ -999,10 +999,6 @@ default="true" class="solr.XMLResponseWriter" /> - - - - --> diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml index c8250755dc7..8350831eeb9 100644 --- a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml +++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml @@ -1243,10 +1243,6 @@ via parameters. The below configuration supports hl.method=original and fastVec default="true" class="solr.XMLResponseWriter" /> - - - - --> diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/python.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/python.adoc index f2fbd3bf5c6..0c94ef5606d 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/python.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/python.adoc @@ -16,45 +16,9 @@ // specific language governing permissions and limitations // under the License. -Solr includes an output format specifically for xref:query-guide:response-writers.adoc#python-response-writer[Python Response Writer], but the xref:query-guide:response-writers.adoc#json-response-writer[JSON Response Writer] is a little more robust. +== Python -== Simple Python - -Making a query is a simple matter. -First, tell Python you will need to make HTTP connections. - -[source,python] ----- -from urllib2 import * ----- - -Now open a connection to the server and get a response. -The `wt` query parameter tells Solr to return results in a format that Python can understand. - -[source,python] ----- -connection = urlopen('http://localhost:8983/solr/collection_name/select?q=cheese&wt=python') -response = eval(connection.read()) ----- - -Now interpreting the response is just a matter of pulling out the information that you need. - -[source,python] ----- -print response['response']['numFound'], "documents found." - -# Print the name of each document. - -for document in response['response']['docs']: - print " Name =", document['name'] ----- - -== Python with JSON - -JSON is a more robust response format, and Python has support for it in its standard library since version 2.6. - -Making a query is nearly the same as before. -However, notice that the `wt` query parameter is now `json` (which is also the default if no `wt` parameter is specified), and the response is now digested by `json.load()`. +JSON is the preferred interchange format for Python users, and Python has support for it in its standard library since version 2.6. [source,python] ---- @@ -69,3 +33,5 @@ print response['response']['numFound'], "documents found." for document in response['response']['docs']: print " Name =", document['name'] ---- + +You can even skip the `wt=json` as it's the default writer type. diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/ruby.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/ruby.adoc index 92b71b5a708..7f82e552e84 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/ruby.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/ruby.adoc @@ -16,100 +16,43 @@ // specific language governing permissions and limitations // under the License. -Solr has an optional Ruby response format that extends the xref:query-guide:response-writers.adoc#json-response-writer[JSON Response Writer] to allow the response to be safely eval'd by Ruby's interpreter -This Ruby response format differs from JSON in the following ways: +== Ruby -* Ruby's single quoted strings are used to prevent possible string exploits -** `\` and `'` are the only two characters escaped... -** unicode escapes not used... data is written as raw UTF-8 -* nil used for null -* `\=>` used as the key/value separator in maps - -Here's an example Ruby response from Solr, for a request like `\http://localhost:8983/solr/techproducts/select?q=iPod&wt=ruby&indent=on` (with Solr launching using `bin/solr start -e techproducts`): - -[source,ruby] ----- -{ - 'responseHeader'=>{ - 'status'=>0, - 'QTime'=>0, - 'params'=>{ - 'q'=>'iPod', - 'indent'=>'on', - 'wt'=>'ruby'}}, - 'response'=>{'numFound'=>3,'start'=>0,'docs'=>[ - { - 'id'=>'IW-02', - 'name'=>'iPod & iPod Mini USB 2.0 Cable', - 'manu'=>'Belkin', - 'manu_id_s'=>'belkin', - 'cat'=>['electronics', - 'connector'], - 'features'=>['car power adapter for iPod, white'], - 'weight'=>2.0, - 'price'=>11.5, - 'price_c'=>'11.50,USD', - 'popularity'=>1, - 'inStock'=>false, - 'store'=>'37.7752,-122.4232', - 'manufacturedate_dt'=>'2006-02-14T23:55:59Z', - '_version_'=>1491038048794705920}, - { - 'id'=>'F8V7067-APL-KIT', - 'name'=>'Belkin Mobile Power Cord for iPod w/ Dock', - 'manu'=>'Belkin', - 'manu_id_s'=>'belkin', - 'cat'=>['electronics', - 'connector'], - 'features'=>['car power adapter, white'], - 'weight'=>4.0, - 'price'=>19.95, - 'price_c'=>'19.95,USD', - 'popularity'=>1, - 'inStock'=>false, - 'store'=>'45.18014,-93.87741', - 'manufacturedate_dt'=>'2005-08-01T16:30:25Z', - '_version_'=>1491038048792608768}, - { - 'id'=>'MA147LL/A', - 'name'=>'Apple 60 GB iPod with Video Playback Black', - 'manu'=>'Apple Computer Inc.', - 'manu_id_s'=>'apple', - 'cat'=>['electronics', - 'music'], - 'features'=>['iTunes, Podcasts, Audiobooks', - 'Stores up to 15,000 songs, 25,000 photos, or 150 hours of video', - '2.5-inch, 320x240 color TFT LCD display with LED backlight', - 'Up to 20 hours of battery life', - 'Plays AAC, MP3, WAV, AIFF, Audible, Apple Lossless, H.264 video', - 'Notes, Calendar, Phone book, Hold button, Date display, Photo wallet, Built-in games, JPEG photo playback, Upgradeable firmware, USB 2.0 compatibility, Playback speed control, Rechargeable capability, Battery level indication'], - 'includes'=>'earbud headphones, USB cable', - 'weight'=>5.5, - 'price'=>399.0, - 'price_c'=>'399.00,USD', - 'popularity'=>10, - 'inStock'=>true, - 'store'=>'37.7752,-100.0232', - 'manufacturedate_dt'=>'2005-10-12T08:00:00Z', - '_version_'=>1491038048799948800}] - }} ----- - -Here is a simple example of how one may query Solr using the Ruby response format: +JSON is the preferred interchange format for Ruby users. [source,ruby] ---- require 'net/http' +require 'json' h = Net::HTTP.new('localhost', 8983) -http_response = h.get('/solr/techproducts/select?q=iPod&wt=ruby') -rsp = eval(http_response.body) +http_response = h.get('/solr/techproducts/select?q=iPod&wt=json') -puts 'number of matches = ' + rsp['response']['numFound'].to_s -#print out the name field for each returned document -rsp['response']['docs'].each { |doc| puts 'name field = ' + doc['name'] } +# Parse the JSON response +rsp = JSON.parse(http_response.body) + +puts 'Number of matches = ' + rsp['response']['numFound'].to_s + +# Print out the name field for each returned document +rsp['response']['docs'].each do |doc| + puts 'Name field = ' + doc['name'] +end ---- -For simple interactions with Solr, this may be all you need! -If you are building complex interactions with Solr, then consider the libraries mentioned in the https://cwiki.apache.org/confluence/display/solr/Ruby+Response+Format#RubyResponseFormat-Libraries[Solr Wiki]. +You can even skip the `wt=json` as it's the default writer type. + +If you are building complex interactions with Solr, then consider these libraries: + +=== Active +* link:http://rubygems.org/gems/rsolr[RSolr]: A lightweight, general purpose client library for Solr. + ** link:https://github.com/sunspot/sunspot[Sunspot]: A nice DSL framework for integrating Solr into your models. Built on top of RSolr. +* link:http://projectblacklight.org[Blacklight]: A popular and well-maintained Ruby on Rails framework for building flexible and attractive front-ends to Solr. + +=== Inactive +* link:http://rubygems.org/gems/solr-ruby[Solr-ruby]: Low-level access to Solr from Ruby. +* link:http://rubygems.org/gems/flare[Flare]: A plugin adding faceted browsing, AJAX suggest, and more to Rails controllers. +* link:http://acts_as_solr.railsfreaks.com/[acts_as_solr]: A plugin to add full-text search capabilities using Solr to ActiveRecord models. link:http://www.quarkruby.com/2007/8/12/acts_as_solr-for-search-and-faceting/[nice acts_as_solr tutorial] +* link:http://rubyforge.org/projects/background-solr/[acts_as_background_solr]: An extension to acts_as_solr moving more processing to the background. +* link:http://mojodna.net/searchable/ruby/[Searchable]: A Rails plugin that provides search integration with ActiveRecord. (an alternative to acts_as_solr) +* link:http://delsolr.rubyforge.org/[DelSolr]: A Solr wrapper designed to simplify facet and filter queries. diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc index e397ccf9db9..be9060e6230 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc @@ -195,7 +195,6 @@ POSTing file sample.html (text/html) to [base]/extract POSTing file sd500.xml (application/xml) to [base] POSTing file solr-word.pdf (application/pdf) to [base]/extract POSTing file solr.xml (application/xml) to [base] -POSTing file test_utf8.sh (application/octet-stream) to [base]/extract POSTing file utf8-example.xml (application/xml) to [base] POSTing file vidcard.xml (application/xml) to [base] 20 files indexed. diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc index 14b26ed556d..3d4e1509fbe 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc @@ -603,7 +603,7 @@ The output includes XML generated by Tika (and further escaped by Solr's XML) us [,console] ---- -$ bin/solr post -c gettingstarted --params "extractOnly=true&wt=ruby&indent=true" --out yes example/exampledocs/sample.html +$ bin/solr post -c gettingstarted --params "extractOnly=true&wt=json&indent=true" --out yes example/exampledocs/sample.html ---- === Using Solr Cell with a POST Request diff --git a/solr/solr-ref-guide/modules/query-guide/pages/response-writers.adoc b/solr/solr-ref-guide/modules/query-guide/pages/response-writers.adoc index d4961312f34..84e938da5fb 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/response-writers.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/response-writers.adoc @@ -24,14 +24,11 @@ Solr supports a variety of Response Writers to ensure that query responses can b The `wt` parameter selects the Response Writer to be used. The list below describe shows the most common settings for the `wt` parameter, with links to further sections that discuss them in more detail. +* <> * <> * <> * <> * <> -* <> -* <> -* <> -* <> * <> * <> * <> @@ -285,67 +282,6 @@ Returns Solr results in http://geojson.org[GeoJSON] augmented with Solr-specific To use this, set `wt=geojson` and `geojson.field` to the name of a spatial Solr field. Not all spatial fields types are supported, and you'll get an error if you use an unsupported one. -== Python Response Writer - -Solr has an optional Python response format that extends its JSON output in the following ways to allow the response to be safely evaluated by the python interpreter: - -* true and false changed to True and False -* Python unicode strings are used where needed -* ASCII output (with unicode escapes) is used for less error-prone interoperability -* newlines are escaped -* null changed to None - -[[php-writer]] -== PHP Response Writer and PHP Serialized Response Writer - -Solr has a PHP response format that outputs an array (as PHP code) which can be evaluated. -Setting the `wt` parameter to `php` invokes the PHP Response Writer. - -Example usage: - -[source,php] ----- -$code = file_get_contents('http://localhost:8983/solr/techproducts/select?q=iPod&wt=php'); -eval("$result = " . $code . ";"); -print_r($result); ----- - -Solr also includes a PHP Serialized Response Writer that formats output in a serialized array. -Setting the `wt` parameter to `phps` invokes the PHP Serialized Response Writer. - -Example usage: - -[source,php] ----- -$serializedResult = file_get_contents('http://localhost:8983/solr/techproducts/select?q=iPod&wt=phps'); -$result = unserialize($serializedResult); -print_r($result); ----- - -== Ruby Response Writer - -Solr has an optional Ruby response format that extends its JSON output in the following ways to allow the response to be safely evaluated by Ruby's interpreter: - -* Ruby's single quoted strings are used to prevent possible string exploits. -* \ and ' are the only two characters escaped. -* Unicode escapes are not used. -Data is written as raw UTF-8. -* nil used for null. -* \=> is used as the key/value separator in maps. - -Here is a simple example of how one may query Solr using the Ruby response format: - -[source,ruby] ----- -require 'net/http' -h = Net::HTTP.new('localhost', 8983) -hresp, data = h.get('/solr/techproducts/select?q=iPod&wt=ruby', nil) -rsp = eval(data) -puts 'number of matches = ' + rsp['response']['numFound'].to_s -#print out the name field for each returned document -rsp['response']['docs'].each { |doc| puts 'name field = ' + doc['name'\] } ----- - == CSV Response Writer The CSV response writer returns a list of documents in comma-separated values (CSV) format. diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc index 532c13fabe0..0f86500fac7 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc @@ -64,3 +64,6 @@ has been removed. Please use `-Dsolr.hiddenSysProps` or the envVar `SOLR_HIDDEN_ * The `BlobRepository`, which was deprecated in 8x in favour of the `FileStore` approach is removed. Users should migrate to the `FileStore` implementation (per node stored file) and the still existing `BlobHandler` (across the cluster storage backed by `.system` collection). Please note this also removes the ability to share resource intensive objects across multiple cores as this feature was tied to the `BlobRepository` implementation. + +* The language specific Response Writers, which were deprecated in 9.8 in favour of more widely used formats like JSON have been removed. +The removed writer types (invoked as part of the `wt` parameter) include `python`, `ruby`, `php`, and `phps`.