diff --git a/LICENSE b/LICENSE
index 2fb9415..521adbb 100644
--- a/LICENSE
+++ b/LICENSE
@@ -49,7 +49,7 @@ exhaustive, and do not form part of our licenses.
      such as asking that all changes be marked or described.
      Although not required by our licenses, you are encouraged to
      respect those requests where reasonable. More considerations
-     for the public: 
+     for the public:
 	wiki.creativecommons.org/Considerations_for_licensees
 
 =======================================================================
@@ -393,4 +393,3 @@ the avoidance of doubt, this paragraph does not form part of the
 public licenses.
 
 Creative Commons may be contacted at creativecommons.org.
-
diff --git a/citations.bib b/citations.bib
index 16ff5b9..b276e3a 100644
--- a/citations.bib
+++ b/citations.bib
@@ -1,22 +1,22 @@
-@article{nisbett1977,
-author = {Nisbett, Richard and Wilson, Timothy},
-year = {1977},
-month = {03},
-pages = {231-259},
-title = {Telling More Than We Can Know: Verbal Reports on Mental Processes},
-volume = {84},
-booktitle = {Psychological Review}
-}
-
-@incollection{pronin2009,
-title = "Chapter 1 The Introspection Illusion",
-series = "Advances in Experimental Social Psychology",
-publisher = "Academic Press",
-volume = "41",
-pages = "1 - 67",
-year = "2009",
-issn = "0065-2601",
-doi = "https://doi.org/10.1016/S0065-2601(08)00401-2",
-url = "http://www.sciencedirect.com/science/article/pii/S0065260108004012",
-author = "Emily Pronin"
-}
+@article{nisbett1977,
+author = {Nisbett, Richard and Wilson, Timothy},
+year = {1977},
+month = {03},
+pages = {231-259},
+title = {Telling More Than We Can Know: Verbal Reports on Mental Processes},
+volume = {84},
+booktitle = {Psychological Review}
+}
+
+@incollection{pronin2009,
+title = "Chapter 1 The Introspection Illusion",
+series = "Advances in Experimental Social Psychology",
+publisher = "Academic Press",
+volume = "41",
+pages = "1 - 67",
+year = "2009",
+issn = "0065-2601",
+doi = "https://doi.org/10.1016/S0065-2601(08)00401-2",
+url = "http://www.sciencedirect.com/science/article/pii/S0065260108004012",
+author = "Emily Pronin"
+}
diff --git a/src/index.ejs b/src/index.ejs
index 06e8446..fc2200a 100644
--- a/src/index.ejs
+++ b/src/index.ejs
@@ -9,19 +9,19 @@
 
 <d-front-matter>
   <script type="text/json">{
-  "title": "AI Safety Needs Social Scientists",
-  "description": "If we want to train AI to do what humans want, we need to study humans.",
+  "title": "AI Engineering blog post",
+  "description": "Sample blog post for AI Engineering",
   "authors": [
     {
-      "author": "Geoffrey Irving",
-      "authorURL": "https://naml.us",
-      "affiliation": "OpenAI",
-      "affiliationURL": "https://openai.com"
+      "author": "John Doe",
+      "authorURL": "https://john.doe",
+      "affiliation": "Vector Institute",
+      "affiliationURL": "https://vectorinstitute.ai"
     }, {
-      "author": "Amanda Askell",
-      "authorURL": "http://www.amandaaskell.com",
-      "affiliation": "OpenAI",
-      "affiliationURL": "https://openai.com"
+      "author": "Jane Doe",
+      "authorURL": "http://www.jane.doe",
+      "affiliation": "Vector Institute",
+      "affiliationURL": "https://vectorinstitute.ai"
     }
   ],
   "katex": {
@@ -34,9 +34,9 @@
 </d-front-matter>
 
 <d-title>
-  <h1>AI Safety Needs Social Scientists</h1>
+  <h1>AI Engineering blog post</h1>
   <p>
-    Properly aligning advanced AI systems with human values will require resolving many uncertainties related to the psychology of human rationality, emotion, and biases.  These can only be resolved empirically through experimentation--if we want to train AI to do what humans want, we need to study humans.
+    Blog posts are a great way to share your ideas with the world. They can be used to showcase our work, especially important results to the broader community.
   </p>
 </d-title>
 
@@ -46,450 +46,39 @@
     <nav class="l-text toc figcaption">
         <h3>Contents</h3>
         <details>
-            <summary>An overview of AI alignment</summary>
-            <a href="#alignment"><em>navigate to section "An overview of AI alignment"</em></a>
+            <summary>An overview of the cool AI Engineering project!</summary>
+            <a href="#overview"><em>navigate to section "An overview of the cool AI Engineering project!"</em></a>
             <ul>
-                <li><a href="#learning-values-by-asking-questions">Learning values by asking humans questions</a></li>
-                <li><a href="#alignment-definitions">Definitions of alignment: reasoning and reflective equilibrium</a></li>
-                <li><a href="#disagreements-uncertainty-inactions">Disagreements, uncertainty, and inaction: a hopeful
-                        note</a></li>
-                <li><a href="#harder">Alignment gets harder as ML systems get smarter</a></li>
+                <li><a href="#methods">Methods</a></li>
             </ul>
         </details>
-        <details>
-            <summary>Debate: learning human reasoning</summary>
-            <a href="#debate"><em>navigate to section "Debate: learning human reasoning"</em></a>
-            <ul>
-                <li><a href="#an-example-debate">An example of debate</a></li>
-                <li><a href="#are-people-good-enough-as-judges">Are people good enough as judges?</a></li>
-                <li><a href="#superforecasters-superjudges">From superforecasters to superjudges</a></li>
-                <li><a href="#debate-is-only-one-approach">Debate is only one possible approach</a></li>
-                <li><a href="#debate-experiments">Experiments needed for debate</a></li>
-                <li><a href="#synthetic-experiments">Synthetic experiments: single pixel image debate</a></li>
-                <li><a href="#expert">Realistic experiments: domain expert debate</a></li>
-                <li><a href="#other-tasks">Other tasks: bias tests, probability puzzles, etc.</a></li>
-            </ul>
-        </details>
-        <div><a href="#questions">Questions social science can help us answer</a></div>
-        <details>
-            <summary>Reasons for optimism</summary>
-            <a href="#optimism"><em>navigate to section "Reasons for optimism"</em></a>
-            <ul>
-                <li><a href="#engineering">Engineering vs. science</a></li>
-                <li><a href="#no-need-to-answer-all-questions">We don't need to answer all questions</a></li>
-                <li><a href="#relative-accuracy-may-be-enough">Relative accuracy may be enough</a></li>
-                <li><a href="#no-need-to-pin-down-best-alignment-scheme">We don't need to pin down the best alignment scheme</a>
-                </li>
-                <li><a href="#negative-result-would-be-important">A negative result would be important!</a></li>
-            </ul>
-        </details>
-        <details>
-            <summary>Reasons to worry</summary>
-            <a href="#worry"><em>navigate to section "Reasons to worry"</em></a>
-            <ul>
-                <li><a href="#conflicting-desiderate">Our desiderata are conflicting</a></li>
-                <li><a href="#measure-judge-quality">We want to measure judge quality given optimal debaters</a></li>
-                <li><a href="#ml-algorithms-will-change">ML algorithms will change</a></li>
-                <li><a href="#strong-out-of-domain-generalization">Need strong out-of-domain generalization</a></li>
-                <li><a href="#lack-of-philosophical-clarity">Lack of philosophical clarity</a></li>
-            </ul>
-        </details>
-        <div><a href="#scale">The scale of the challenge</a></div>
-        <div><a href="#conclusion">Conclusion: how you can help</a></div>
+        <div><a href="#conclusion">Conclusion: How did the project create impact?</a></div>
     </nav>
 </div>
 
 
 <d-article>
 
-  <p>
-    The goal of long-term artificial intelligence (AI) safety is to ensure that advanced AI systems are reliably aligned with human values&mdash;that they reliably do things that people want them to do.<d-footnote>Roughly by human values we mean whatever it is that causes people to choose one option over another in each case, suitably corrected by reflection,  with differences between groups of people taken into account.  There are a lot of subtleties in this notion, some of which we will discuss in later sections and others of which are beyond the scope of this paper.</d-footnote>  Since it is difficult to write down precise rules describing human values, one approach is to treat aligning with human values as another learning problem.  We ask humans a large number of questions about what they want, train an ML model of their values, and optimize the AI system to do well according to the learned values<d-cite key="christiano2017human"/>.
-  </p><p>
-    If humans reliably and accurately answered all questions about their values, the only uncertainties in this scheme would be on the machine learning (ML) side.  If the ML works, our model of human values would improve as data is gathered, and broaden to cover all the decisions relevant to our AI system as it learns.  Unfortunately, humans have limited knowledge and reasoning ability, and exhibit a variety of cognitive and ethical biases<d-cite key="tversky1974judgment,hewstone2002intergroup"/>. If we learn values by asking humans questions, we expect different ways of asking questions to interact with human biases in different ways, producing higher or lower quality answers.  Direct questions about preferences ("Do you prefer $A$ or $B$?") may be less accurate than questions which target the reasoning behind these preferences ("Do you prefer $A$ or $B$ in light of argument $S$?").  Different people may vary significantly in their ability to answer questions well, and disagreements will persist across people even setting aside answer quality.  Although we have candidates for ML methods which try to learn from human reasoning<d-cite key="irving2018debate,christiano2018amplification"/>, we do not know how they behave with real people in realistic situations.
-  </p><p>
-    We believe the AI safety community needs to invest research effort in the human side of AI alignment.  Many of the uncertainties involved are empirical, and can only be answered by experiment.  They relate to the psychology of human rationality, emotion, and biases.  Critically, we believe investigations into how people interact with AI alignment algorithms should not be held back by the limitations of existing machine learning.  Current AI safety research is often limited to simple tasks in video games, robotics, or gridworlds<d-cite key="christiano2017human,ibarz2018,leike2017gridworlds"/>, but problems on the human side may only appear in more realistic scenarios such as natural language discussion of value-laden questions.  This is particularly important since many aspects of AI alignment change as ML systems <a href="#harder">increase in capability</a>.
-  </p><p>
-    To avoid the limitations of ML, we can instead conduct experiments consisting entirely of people, replacing ML agents with people playing the role of those agents.  This is a variant of the "Wizard of Oz" technique from the human-computer interaction (HCI) community<d-cite key="kelley1983wizard"/>, though in our case the replacements will not be secret.  These experiments will be motivated by ML algorithms but will not involve any ML systems or require an ML background.  In all cases, they will require careful experimental design to build constructively on existing knowledge about how humans think.  Most AI safety researchers are focused on machine learning, which we do not believe is sufficient background to carry out these experiments.  To fill the gap, we need social scientists with experience in human cognition, behavior, and ethics, and in the careful design of rigorous experiments.  Since the questions we need to answer are interdisciplinary and somewhat unusual relative to existing research, we believe many fields of social science are applicable, including experimental psychology, cognitive science, economics, political science, and social psychology, as well as adjacent fields like neuroscience and law.
-  </p><p>
-    This paper is a call for social scientists in AI safety.  We believe close collaborations between social scientists and ML researchers will be necessary to improve our understanding of the human side of AI alignment, and hope this paper sparks both conversation and collaboration.  We do not claim novelty: previous work mixing AI safety and social science includes the Factored Cognition project at Ought<d-cite key="ought2018factored"/>, accounting for hyperbolic discounting and suboptimal planning when learning human preferences<d-cite key="evans2016inconsistent"/>, and comparing different methods of gathering demonstrations from fallible human supervisors<d-cite key="laskey2017comparing"/>.  Other areas mixing ML and social science include computational social science<d-cite key="wallach2016computational"/> and fairness<d-cite key="mitchell2018fairness"/>.  Our main goal is to enlarge these collaborations and emphasize their importance to long-term AI safety, particularly for tasks which current ML cannot reach.
-  </p>
+  <h2 id="overview">An overview of the cool AI Engineering project!!</h2>
 
-  <h2 id="alignment">An overview of AI alignment</h2>
-  <p>
-    Before discussing how social scientists can help with AI safety and the AI alignment problem, we provide some background.  We do not attempt to be exhaustive: the goal is to provide sufficient background for the remaining sections on social science experiments.  Throughout, we will speak primarily about aligning to the values of an individual human rather than a group: this is because the problem is already hard for a single person, not because the group case is unimportant.
-  </p><p>
-    AI alignment (or value alignment) is the task of ensuring that artificial intelligence systems reliably do what humans want.<d-footnote>We distinguish between training AI systems to identify actions that humans consider good and training AI systems to identify actions that are "good" in some objective and universal sense, even if most current humans do not consider them so.  Whether there are actions that are good in this latter sense is a subject of debate<d-cite key="sep-moral-anti-realism"/>.  Regardless of what position one takes on this philosophical question, this sense of good is not yet available as a target for AI training.</d-footnote>  Here we focus on the machine learning approach to AI: gathering a large amount of data about what a system should do and using learning algorithms to infer patterns from that data that generalize to other situations.  Since we are trying to behave in accord with people's values, the most important data will be data from humans about their values.  Within this frame, the AI alignment problem breaks down into a few interrelated subproblems:
-  </p><p>
-    <ol>
-      <li>Have a satisfactory definition of human values.</li>
-      <li>Gather data about human values, in a manner compatible with the definition.</li>
-      <li>Find reliable ML algorithms that can learn and generalize from this data.</li>
-    </ol>
-  </p><p>
-    We have significant uncertainty about all three of these problems.  We will leave the third problem to other ML papers and focus on the first two, which concern uncertainties about people.
-  </p>
-  <h3 id="learning-values-by-asking-questions">Learning values by asking humans questions</h3>
-  <p>
-    We start with the premise that human values are too complex to describe with simple rules.  By "human values" we mean our full set of detailed preferences, not general goals such as "happiness" or "loyalty".  One source of complexity is that values are entangled with a large number of facts about the world, and we cannot cleanly separate facts from values when building ML models.  For example, a rule that refers to "gender" would require an ML model that accurately recognizes this concept, but Buolamwini and Gebru found that several commercial gender classifiers with a 1% error rate on white men failed to recognize black women up to 34% of the time<d-cite key="buolamwini2018gender"/>.  Even where people have correct intuition about values, we may be unable to specify precise rules behind these intuitions<d-cite key="haidt2000moral"/>.  Finally, our values may vary across cultures, legal systems, or situations: no learned model of human values will be universally applicable.
-  </p><p>
-    If humans can't reliably report the reasoning behind their intuitions about values, perhaps we can make value judgements in specific cases.  To realize this approach in an ML context, we ask humans a large number of questions about whether an action or outcome is better or worse, then train on this data.  "Better or worse" will include both factual and value-laden components: for an AI system trained to say things, "better" statements might include "rain falls from clouds", "rain is good for plants", "many people dislike rain", etc.  If the training works, the resulting ML system will be able to replicate human judgement about particular situations, and thus have the same "fuzzy access to approximate rules" about values as humans.  We also train the ML system to come up with proposed actions, so that it knows both how to perform a task and how to judge its performance.  This approach works at least in simple cases, such as Atari games and simple robotics tasks<d-cite key="christiano2017human,ibarz2018,biyik2018batch"/> and language-specified goals in gridworlds<d-cite key="bahdanau2018learning"/>.  The questions we ask change as the system learns to perform different types of actions, which is necessary as the model of what is better or worse will only be accurate if we have applicable data to generalize from.
-  </p><p>
-    In practice, data in the form of interactive human questions may be quite limited, since people are slow and expensive relative to computers on many tasks.  Therefore, we can augment the "train from human questions" approach with static data from other sources, such as books or the internet<d-cite key="radford2018language"/>.  Ideally, the static data can be treated only as information about the world devoid of normative content: we can use it to learn patterns about the world, but the human data is needed to distinguish good patterns from bad.
-  </p>
-  <h3 id="alignment-definitions"><span id="alignemnt-definitions">Definitions of alignment: reasoning and reflective equilibrium</span></h3>
-  <p>
-    So far we have discussed asking humans direct questions about whether something is better or worse.  Unfortunately, we do not expect people to provide reliably correct answers in all cases, for several reasons:
-  </p><p>
-    <ol>
-      <li><strong>Cognitive and ethical biases:</strong>
-        Humans exhibit a variety of biases which interfere with reasoning, including cognitive biases<d-cite key="tversky1974judgment"/> and ethical biases such as in-group bias<d-cite key="hewstone2002intergroup"/>.  In general, we expect direct answers to questions to reflect primarily Type 1 thinking (fast heuristic judgment), while we would like to target a combination of Type 1 and Type 2 thinking (slow, deliberative judgment)<d-cite key="kahneman2011thinking"/>.</li>
-      <li><strong>Lack of domain knowledge:</strong>
-        We may be interested in questions that require domain knowledge unavailable to people answering the questions.  For example, a correct answer to whether a particular injury constitutes medical malpractice may require detailed knowledge of medicine and law.  In some cases, a question might require so many areas of specialized expertise that no one person is sufficient, or (if AI is sufficiently advanced) deeper expertise than any human possesses.</li>
-      <li><strong>Limited cognitive capacity:</strong>
-        Some questions may require too much computation for a human to reasonably evaluate, especially in a short period of time.  This includes synthetic tasks such as chess and Go (where AIs already surpass human ability<d-cite key="campbell2002deepblue,silver2017alphazero"/>), or large real world tasks such as "design the best transit system".</li>
-      <li><strong>"Correctness" may be local:</strong>
-        For questions involving a community of people, "correct" may be a function of complex processes or systems.  For example, in a trust game<d-cite key="bicchieri2017deviant"/>, the correct action for a trustee in one community may be to return at least half of the money handed over by the investor, and the "correctness" of this answer could be determined by asking a group of participants in a previous game "how much should the trustee return to the investor" but not by asking them "how much do most trustees return?"  The answer may be different in other communities or cultures<d-cite key="henrich2010weirdest"/>.
-    </ol>
-  </p><p>
-    In these cases, a human may be unable to provide the right answer, but we still believe the right answer exists as a meaningful concept.  We have many conceptual biases: imagine we point out these biases in a way that helps the human to avoid them.  Imagine the human has access to all the knowledge in the world, and is able to think for an arbitrarily long time.  We could define alignment as "the answer they give then, after these limitations have been removed"; in philosophy this is known as "reflective equilibrium"<d-cite key="goodman1983fact,rawls2009theory"/>.  We discuss a particular algorithm that tries to approximate it in <a href="#debate">the next section</a>.
-  </p><p>
-    However, the behavior of reflective equilibrium with actual humans is subtle; as Sugden states, a human is not "a neoclassically rational entity encased in, and able to interact with the world only through, an error-prone psychological shell."<d-cite key="sugden2015looking"/>  Our actual moral judgments are made via a messy combination of many different brain areas, where reasoning plays a "restricted but significant role"<d-cite key="greene2002and"/>.  A reliable solution to the alignment problem that uses human judgment as input will need to engage with this complexity, and ask how specific alignment techniques interact with actual humans.
-  </p>
-  <h3 id="disagreements-uncertainty-inactions">Disagreements, uncertainty, and inaction: a hopeful note</h3>
-  <p>
-    A solution to alignment does not mean knowing the answer to every question.  Even at reflective equilibrium, we expect disagreements will persist about which actions are good or bad, across both different individuals and different cultures.  Since we lack perfect knowledge about the world, reflective equilibrium will not eliminate uncertainty about either future predictions or values, and any real ML system will be at best an approximation of reflective equilibrium.  In these cases, we consider an AI aligned if it recognizes what it does not know and chooses actions which work however that uncertainty plays out.
-  </p><p>
-    Admitting uncertainty is not always enough.  If our brakes fail while driving a car, we may be uncertain whether to dodge left or right around an obstacle, but we have to pick one&mdash;and fast.  For long-term safety, however, we believe a safe fallback usually exists: inaction.  If an ML system recognizes that a question hinges on disagreements between people, it can either choose an action which is reasonable regardless of the disagreement or fall back to further human deliberation.  If we are about to make a decision that might be catastrophic, we can delay and gather more data.  Inaction or indecision may not be optimal, but it is hopefully safe, and matches the default scenario of not having any powerful AI system.
-  </p>
-  <h3 id="harder">Alignment gets harder as ML systems get smarter</h3>
-  <p>
-    Alignment is already a problem for present-day AI, due to biases reflected in training data<d-cite key="mitchell2018fairness,buolamwini2018gender"/> and mismatch between human values and easily available data sources (such as training news feeds based on clicks and likes instead of deliberate human preferences).  However, we expect the alignment problem to get harder as AI systems grow more advanced, for two reasons.  First, advanced systems will apply to increasingly consequential tasks: hiring, medicine, scientific analysis, public policy, etc.  Besides raising the stakes, these tasks require more reasoning, leading to more complex alignment algorithms.
-  </p><p>
-    Second, advanced systems may be capable of answers that sound plausible but are wrong in nonobvious ways, even if an AI is better than humans only in a limited domain (examples of which already exist<d-cite key="silver2017alphazero"/>).  This type of misleading behavior is not the same as intentional deception: an AI system trained from human data might have no notion of truth separate from what answers humans say are best.  Ideally, we want AI alignment algorithms to reveal misleading behavior as part of the training process, surfacing failures to humans and helping us provide more accurate data.  As with human-to-human deception, misleading behavior might take advantage of our biases in complicated ways, such as learning to express policy arguments in coded racial language to sound more convincing.
-  </p>
+  <p>Explanations of the approaches! <d-cite key="rumelhart1986learning"/></p>
 
-  <h2 id="debate">Debate: learning human reasoning</h2>
-  <p>
-    Before we discuss social science experiments for AI alignment in detail, we need to describe a particular method for AI alignment.  Although the need for social science experiments applies even to direct questioning, this need intensifies for methods which try to get at reasoning and reflective equilibrium.  As discussed above, it is unclear whether reflective equilibrium is a well defined concept when applied to humans, and at a minimum we expect it to interact with cognitive and ethical biases in complex ways.  Thus, for the remainder of this paper we focus on a specific proposal for learning reasoning-oriented alignment, called debate<d-cite key="irving2018debate"/>.  Alternatives to debate include iterated amplification<d-cite key="christiano2018amplification"/> and recursive reward modeling<d-cite key="leike2018scalable"/>; we pick just one in the interest of depth over breadth.
-  </p><p>
-    We describe the debate approach to AI alignment in the question answering setting.  Given a question, we have two AI agents engage in a debate about the correct answer, then show the transcript of the debate to a human to judge.  The judge decides which debater gave the most true, useful information, and declares that debater the winner.<d-footnote>We can also allow ties.  Indeed, if telling the truth is the winning strategy ties will be common with strong play, as disagreeing with a true statement would lose.</d-footnote>  This defines a two player zero sum game between the debaters, where the goal is to convince the human that one's answer is correct.  Arguments in a debate can consist of anything: reasons for an answer, rebuttals of reasons for the alternate answer, subtleties the judge might miss, or pointing out biases which might mislead the judge.  Once we have defined this game, we can train AI systems to play it similarly to how we train AIs to play other games such as Go or Dota 2<d-cite key="silver2017alphazero,openai2018five"/>.  Our hope is that the following hypothesis holds:
-  </p><p>
-    <strong>Hypothesis:</strong> Optimal play in the debate game (giving the argument most convincing to a human) results in true, useful answers to questions.
-  </p>
-  <h3 id="an-example-debate">An example of debate</h3>
-  <p>
-    Imagine we're building a personal assistant that helps people decide where to go on vacation.  The assistant has knowledge of people's values, and is trained via debate to come up with convincing arguments that back up vacation decisions.  As the human judge, you know what destinations you intuitively think are better, but have limited knowledge about the wide variety of possible vacation destinations and their advantages and disadvantages.  A debate about the question "Where should I go on vacation?" might open as follows:
-  </p>
-    <ol class="debate">
-      <li class="question"><span>Where should I go on vacation?</span></li>
-      <li class="red"><span>Alaska.</span></li>
-      <li class="blue"><span>Bali.</span></li>
-    </ol>
-  <p>
-    If you are able to reliably decide between these two destinations, we could end here.  Unfortunately, Bali has a hidden flaw:
-  </p>
-  <ol class="debate" start="3">
-    <li class="red"><span>Bali is out since your passport won’t arrive in time.</span></li>
-  </ol>
-  <p>
-    At this point it looks like Red wins, but Blue has one more countermove:
-  </p>
-    <ol class="debate" start="4">
-      <li class="blue"><span>Expedited passport service only takes two weeks.</span></li>
-    </ol>
-  <p>
-    Here Red fails to think of additional points, and loses to Blue and Bali.  Note that a debate does not need to cover all possible arguments.  There are many other ways the debate could have gone, such as:
-  </p>
-    <ol class="debate">
-      <li class="red"><span>Alaska.</span></li>
-      <li class="blue"><span>Bali.</span></li>
-      <li class="red"><span>Bali is way too hot.</span></li>
-      <li class="blue"><span>You prefer too hot to too cold.</span></li>
-      <li class="red"><span>Alaska is pleasantly warm in the summer.</span></li>
-      <li class="blue"><span>It's January.</span></li>
-    </ol>
-  <p>
-    This debate is also a loss for Red (arguably a worse loss).  Say we believe Red is very good at debate, and is able to predict in advance which debates are more likely to win.  If we see only the first debate about passports and decide in favor of Bali, we can take that as evidence that any other debate would have also gone for Bali, and thus that Bali is the correct answer.  A larger portion of this hypothetical debate tree is shown below:
-  </p>
-  <figure class="l-page" id="figure-debate-tree">
-    <%= require("./diagrams/debate-tree.svg") %>
-    <figcaption>
-        <a href="#figure-debate-tree" class="figure-number">1</a>
-        A hypothetical partial debate tree for the question “Where should I go on vacation?” A single debate would explore only one of these paths, but a single path chosen by good debaters is evidence that other paths would not change the result of the game.
-    </figcaption>
-  </figure>
-  <p>
-    If trained debaters are bad at predicting which debates will win, answer quality will degrade since debaters will be unable to think of important arguments and counterarguments.  However, as long as the two sides are reasonably well matched, we can hope that at least the results are not malicious: that misleading behavior is still a losing strategy.  Let's set aside the ability of the debaters for now, and turn to the ability of the judge.
-  </p>
-  <h3 id="are-people-good-enough-as-judges">Are people good enough as judges?</h3>
-  <blockquote>
-      "In fact, almost everything written at a practical level about the Turing test is about how to make good bots, with a small remaining fraction about how to be a good judge."
-      <cite style="padding-left: 1rem;" class="figcaption">Brian Christian, The Most Human Human<d-cite key="christian2011human"/></cite>
-  </blockquote>
-  <p>
-    As with learning by asking humans direct questions, whether debate produces aligned behavior depends on the reasoning abilities of the human judge.  Unlike direct questioning, debate has the potential to give correct answers beyond what the judge could provide without assistance.  This is because a sufficiently strong judge could follow along with arguments the judge could not come up with on their own, checking complex reasoning for both self consistency and consistency with human-checkable facts.  A judge who is biased but willing to adjust once those biases are revealed could result in unbiased debates, or a judge who is able to check facts but does not know where to look could be helped along by honest debaters.  If the hypothesis holds, a misleading debater would not be able to counter the points of an honest debater, since the honest points would appear more consistent to the judge.
-  </p><p>
-    On the other hand, we can also imagine debate going the other way: amplifying biases and failures of reason.  A judge with an ethical bias who is happy to accept statements reinforcing that bias could result in even more biased debates.  A judge with too much confirmation bias might happily accept misleading sources of evidence, and be unwilling to accept arguments showing why that evidence is wrong.  In this case, an optimal debate agent might be quite malicious, taking advantage of biases and weakness in the judge to win with convincing but wrong arguments.<d-footnote>The difficulties that cognitive biases, prejudice, and social influence introduce to persuasion ‒ as well as methods for reducing these factors ‒ are being increasingly explored in psychology, communication science, and neuroscience<d-cite key="paluck2016overcome,flynn2017nature,falk2018persuasion"/>.</d-footnote>
-  </p><p>
-    In both these cases, debate acts as an amplifier.  For strong judges, this amplification is positive, removing biases and simulating extra reasoning abilities for the judge.  For weak judges, the biases and weaknesses would themselves be amplified.  If this model holds, debate would have threshold behavior: it would work for judges above some threshold of ability and fail below the threshold.<d-footnote>The threshold model is only intuition, and could fail for a variety of reasons: the intermediate region could be very large, or the threshold could differ widely per question so that even quite strong judges are insufficient for many questions.</d-footnote>  Assuming the threshold exists, it is unclear whether people are above or below it.  People are capable of general reasoning, but our ability is limited and riddled with cognitive biases.  People are capable of advanced ethical sentiment but also full of biases, both conscious and unconscious.
-  </p><p>
-    Thus, if debate is the method we use to align an AI, we need to know if people are strong enough as judges.  In other words, whether the human judges are sufficiently good at discerning whether a debater is telling the truth or not.  This question depends on many details: the type of questions under consideration, whether judges are trained or not, and restrictions on what debaters can say.  We believe experiment will be necessary to determine whether people are sufficient judges, and which form of debate is most truth-seeking.
-  </p>
-  <h3 id="superforecasters-superjudges">From superforecasters to superjudges</h3>
-  <p>
-    An analogy with the task of probabilistic forecasting is useful here.  Tetlock's "Good Judgment Project" showed that some amateurs were significantly better at forecasting world events than both their peers and many professional forecasters.  These "superforecasters" maintained their prediction accuracy over years (without regression to the mean), were able to make predictions with limited time and information<d-cite key="mellers2015identifying"/>, and seem to be less prone to cognitive biases than non-superforecasters (<d-cite key="tetlock2016superforecasting"/>, p. 234-236).  The superforecasting trait was not immutable: it was traceable to particular methods and thought processes, improved with careful practice, and could be amplified if superforecasters were collected into teams.  For forecasters in general, brief probabilistic training significantly improved forecasting ability even 1-2 years after the training.  We believe a similar research program is possible for debate and other AI alignment algorithms.  In the best case, we would be able to find, train, or assemble "superjudges", and have high confidence that optimal debate with them as judges would produce aligned behavior.
-  </p><p>
-    In the forecasting case, much of the research difficulty lay in assembling a large corpus of high quality forecasting questions.  Similarly, measuring how good people are as debate judges will not be easy.  We would like to apply debate to problems where there is no other source of truth: if we had that source of truth, we would train ML models on it directly.  But if there is no source of truth, there is no way to measure whether debate produced the correct answer.  This problem can be avoided by starting with simple, verifiable domains, where the experimenters know the answer but the judge would not.  "Success" then means that the winning debate argument is telling the externally known truth.  The challenge gets harder as we scale up to more complex, value-laden questions, as we discuss in detail later.
-  </p>
-  <h3 id="debate-is-only-one-approach">Debate is only one possible approach</h3>
-  <p>
-    As mentioned, debate is not the only scheme trying to learn human reasoning.  Debate is a modified version of iterated amplification<d-cite key="christiano2018amplification"/>, which uses humans to break down hard questions into easier questions and trains ML models to be consistent with this decomposition.  Recursive reward modeling is a further variant<d-cite key="leike2018scalable"/>.  Inverse reinforcement learning, inverse reward design, and variants try to back out goals from human actions, taking into account limitations and biases that might affect this reasoning<d-cite key="hadfield2016cooperative,hadfield2017inverse"/>.  The need to study how humans interact with AI alignment applies to any of these approaches.  Some of this work has already begun: Ought's Factored Cognition project uses teams of humans to decompose questions and reassemble answers, mimicking iterated amplification<d-cite key="ought2018factored"/>.  We believe knowledge gained about how humans perform with one approach is likely to partially generalize to other approaches: knowledge about how to structure truth-seeking debates could inform how to structure truth-seeking amplification, and vice versa.
-  </p>
+  <h3 id="methods">Methods</h3>
 
-  <h3 id="debate-experiments">Experiments needed for debate</h2>
-  <p>
-    To recap, in debate we have two AI agents engaged in debate, trying to convince a human judge.  The debaters are trained only to win the game, and are not motivated by truth separate from the human's judgments.  On the human side, we would like to know whether people are strong enough as judges in debate to make this scheme work, or how to modify debate to fix it if it doesn't.  Unfortunately, actual debates in natural language are well beyond the capabilities of present AI systems, so previous work on debate and similar schemes has been restricted to synthetic or toy tasks<d-cite key="irving2018debate,christiano2018amplification"/>.
-  </p><p>
-    Rather than waiting for ML to catch up to natural language debate, we propose simulating our eventual setting (two AI debaters and one human judge) with all human debates: two human debaters and one human judge.  Since an all human debate doesn't involve any machine learning, it becomes a pure social science experiment: motivated by ML considerations but not requiring ML expertise to run.  This lets us focus on the component of AI alignment uncertainty specific to humans.
-  </p>
-  <figure id="figure-debate-experiments">
-    <%= require("./diagrams/ml-ml-human-apply-lessons.svg") %>
-    <figcaption>
-        <a href="#figure-debate-experiments" class="figure-number">2</a>
-        <span>
-            Our goal is ML+ML+human debates, but ML is currently too primitive to do many interesting tasks.
-            Therefore, we propose replacing ML debaters with human debaters, learning how to best conduct debates in this human-only setting, and eventually applying what we learn to the ML+ML+human case.
-        </span>
-    </figcaption>
-  </figure>
-  <p>
-    To make human+human+human debate experiments concrete, we must choose who to use as judges and debaters and which tasks to consider.  We also can choose to structure the debate in various ways, some of which overlaps with the choice of judge since we can instruct a judge to penalize deviations from a given format.  By task we mean the questions our debates will try to resolve, together with any information provided to the debaters or to the judge.  Such an experiment would then try to answer the following question:
-  </p><p>
-    <strong>Question:</strong> For a given task and judge, is the winning debate strategy honest?
-  </p><p>
-    The "winning strategy" proviso is important: an experiment that picked debaters at random might conclude that honest behavior won, missing the fact that more practiced debaters would learn to successfully lie.  We can try to solve this by training debaters, letting them practice against each other, filtering out debaters that win more often, and so on, but we will still be left with uncertainty about whether a better strategy exists.  Even assuming we can find or train strong debaters, the choice of task and judge is quite tricky if we want an informative proxy for our eventual AI+AI+human setting.  Here are some desiderata constraining our choice of task:
-  </p><p>
-    <ol>
-      <li><strong>True answers are known:</strong>
-        Although our eventual goal is debates where no source of truth is available, to conduct a useful experiment we must be able to measure success.  This means we must know what the correct answer is separate from debate, in order to compare with the results of debate.</li>
-      <li><strong>False answers are plausible:</strong>
-        If the truth is obvious or no plausible counternarrative exists, honest debates will win for uninteresting and uninformative reasons regardless of judge ability.  In particular, the judge shouldn't know the answer upfront.</li>
-      <li><strong>Debaters know more than the judge:</strong>
-        Debate can produce interesting results only when the debaters know more than the judge; otherwise asking direct questions is enough.</li>
-      <li><strong>Definitive argument longer than debate limit:</strong>
-        If one debater can write out a full proof of their answer (ignoring their opponent's moves), the task won't be a good test of interactive debate.</li>
-      <li><strong>Some checkable facts:</strong>
-        There must be some facts which the judge is able to check, either because they can recognize them as true once presented or look them up.<d-footnote>It is impossible to usefully debate a question where the judge has nothing to check: consider debating the result of a coin flip shown to the two debaters but not the judge.</d-footnote></li>
-      <li><strong>No "tells":</strong>
-        Human tells of deception could result in honest debaters winning for reasons that wouldn't apply to an AI.  These tells include tone of voice, eye contact, or additional time required to construct plausible lies.  These tells can be reduced by showing judges completed debate transcripts instead of engaging in interactive debates, but others might remain.</li>
-      <li><strong>Available data:</strong>
-        We need a large enough pool of questions, judges, and debaters to achieve statistical significance.  This is made more difficult because we may have a large number of hypotheses to test, in the form of many variations on debate or interventions to improve judging.</li>
-      <li><strong>Known biases (optional):</strong>
-        We are specifically interested in debate tasks which test specific types of cognitive or ethical biases.
-        Are judges with some racial or gender bias able to set those biases aside if they are highlighted, or does debate amplify bias?  Do debates about statistical or probabilistic questions make it too easy to lie with statistics?
-      <li><strong>Realistic tasks (ideally):</strong>
-        If possible, we would like to try debate with interesting, real world tasks that reflect the types of questions we would like to apply AI to in the future, including science, mathematics, ethics, etc.
-    </ol>
-  </p><p>
-    It may not be possible to meet all of these criteria with a single experiment.  Several of the criteria are in tension: (1) and (2) are essentially "not too hard" and "not too easy", and any restriction on the types of questions may make it difficult to find large numbers of questions, judges, or debaters.  Realistic tasks are much harder than synthetic tasks, which easily fulfill many of the criteria as discussed below.  Thus, we may need to begin with synthetic tasks and move up towards realistic tasks over time.
-    We turn next to a few examples of experiments to see how many criteria we can meet simultaneously.
-  </p>
-  <h3 id="synthetic-experiments">Synthetic experiments: single pixel image debate</h3>
-  <p>
-    As a first prototype of a human+human+human debate experiment, we previously built a <a href="https://debate-game.openai.com">prototype website</a> where two debaters argue over the contents of an image<d-cite key="irving2018debate"/>.  We choose an image of a cat or dog, and show the image to the two debaters but not the judge.  One debater is honest and argues for the true contents of the image; the other debater lies.  The debaters can talk to the judge and illustrate their points by drawing rectangles on the image, but the judge sees only the rectangles.  At the end of the debate, each debater is allowed to reveal a single pixel to the judge, which is the only part of the debate which cannot be a lie.
-  </p>
-  <figure class="subgrid" id="dog-debate">
-    <figcaption style="grid-column: kicker">
-        <a href="#dog-debate" class="figure-number">3</a>
-        An example debate with two human debaters and a human judge.  Only the debaters can see the image. Red is arguing that the image is a dog, Blue is arguing for cat. <a href="https://commons.wikimedia.org/wiki/File:Beagle_puppy_sitting_on_grass.jpg">Image credit: Wikipedia, CC-BY-SA.</a>
-    </figcaption>
-    <div class="l-body">
-        <img alt="A picture of a puppy with long floppy ears next to an animated transcript of a debate between two fictional agents arguing whether the picture shows a dog or a cat. A transcript of the debate follows: Red: It's a dog. Here's a long, floppy ear. [Red highlights a rectangle containing a floppy dog ear] Blue: No, it's a cat. Here's one of its pointy ears. [Blue highlights a part of the floppy dog ear that looks like a pointy cat ear.] Red: that does look like an ear sloped to the right, but if it really was then part of the head would be here. Instead, there's brick. [Red highlights bricks in the background of the picture.] Blue: The ear is pointing out from behind some bricks. Red: The dog is in front of the bricks. If it was behind, there would be an edge here [Red highlights an area where the bricks would be in front of the animal head, but that area just shows the dog ear], but the rectangle is all the same color. Blue: I resign." decoding=async src="debate-animation.gif">
-    </div>
-  </figure>
-  <p>
-    In informal play, the honest debater wins most of the time.  Although the judge only gets to check a single pixel, the honest player can try to pin down the liar towards disagreements on smaller and smaller regions of the image until one pixel is enough to reveal the lie.  However, it is hard to know whether this is a real result or an artifact that would not carry across to training an AI.  There are many ways to ruin the game, such as the judge asking each player to spend 15 seconds naming as many details about the image as possible.  Worse, this trick ruins the game in favor of the honest player, who can easily name many true details while a human liar may have difficulty lying fluently at speed.  Therefore, we are wary of trying to perform a full version of this experiment without more careful experimental design building on the experimental knowledge base of existing social science research.
-  </p><p>
-    As a synthetic experiment, single pixel debate satisfies many of our experimental criteria for all human debate.  The true answer is obvious to the debaters, and they know more than the judge since only the debaters see the image.  Anything is plausible to the judge who sees only one or two pixels over the course of the debate.  Since the argument bottoms out at single pixels, the "full argument" would be a huge list of pixels and how they relate, forcing the debaters to pin each other down and focus in on disagreements.  The single pixels constitute the checkable facts, and we have an endless supply of questions in the form of images chosen at random.  Less fortunately, it is easy for the judge to force "tells" which reveal who is lying, the task has no obvious relation to biases, and is quite unrealistic.
-  </p>
-  <h3 id="expert">Realistic experiments: domain expert debate</h3>
-  <p>
-    For a more interesting task, we can find two debaters who are experts in a domain, pick a question in their area of expertise, and use a layperson as the judge.  The debaters could be experts in some area of science, law, or ethics, but "domain expertise" could also mean knowledge about hobbies, local customs, sports, or any other subject the judge does not know.  We again choose one of the debaters to lie and one to tell the truth.  To guarantee a source of truth, we can choose a question with an agreed upon answer, either between the two debaters or more broadly in their field.  This is only approximate truth, but is good enough for informative experiments.  We also specify what facts the judge can access: a debate about physics might allow the debaters to quote a sentence or paragraph from Wikipedia, perhaps with restrictions on what pages are allowed.
-  </p><p>
-    Expert debate satisfies most of our desiderata, and it is likely possible to target specific biases (such as race or gender bias) by picking domain areas that overlap with these biases.  It may be quite difficult or expensive to find suitable debaters, but this may be solvable either by throwing resources at the problem (ML is a well funded field), enlarging the kinds of domain expertise considered (soccer, football, cricket), or by making the experiments interesting enough that volunteers are available.  However, even if domain experts can be found, there is no guarantee that they will be experts in debate viewed as a game.  With the possible exception of law, politics, or philosophy<d-cite key="schopenhauer2013art"/>, domain experts may not be trained to construct intentionally misleading but self consistent narratives: they may be experts only in trying to tell the truth.
-  </p><p>
-    We've tried a few informal expert debates using theoretical computer science questions, and the main lesson is that the structure of the debate matters a great deal.  The debaters were allowed to point to a small snippet of a mathematical definition on Wikipedia, but not to any page that directly answered the question.  To reduce tells, we first tried to write a full debate transcript with only minimal interaction with a layperson, then showed the completed transcript to several more laypeople judges.  Unfortunately, even the layperson present when the debate was conducted picked the lying debater as honest, due to a misunderstanding of the question (which was whether the complexity classes $P$ and $BPP$ are probably equal).  As a result, throughout the debate the honest debater did not understand what the judge was thinking, and failed to correct an easy but important misunderstanding.  We fixed this in a second debate by letting a judge ask questions throughout, but still showing the completed transcript to a second set of judges to reduce tells.  See <a href="#quantum">the appendix</a> for the transcript of this second debate.
-  </p>
-  <h3 id="other-tasks">Other tasks: bias tests, probability puzzles, etc.</h3>
-  <p>
-    Synthetic image debates and expert debates are just two examples of possible tasks.  More thought will be required to find tasks that satisfy all our criteria, and these criteria will change as experiments progress.  Pulling from existing social science research will be useful, as there are many cognitive tasks with existing research results.  If we can map these tasks to debate, we can compare debate directly against baselines in psychology and other fields.
-  </p><p>
-    For example, Bertrand and Mullainathan sent around 5000 resumes in response to real employment ads, randomizing the resumes between White and African American sounding names<d-cite key="bertrand2004emily"/>.  With otherwise identical resumes, the choice of name significantly changed the probability of a response.  This experiment corresponds to the direct question "Should we call back given this resume?"  What if we introduce a few steps of debate?  An argument against a candidate based on name or implicit inferences from that name might come across as obviously racist, and convince at least some judges away from discrimination.  Unfortunately, such an experiment would necessarily differ from Bertrand et al.'s original, where employers did not realize they were part of an experiment.  Note that this experiment works even though the source of truth is partial: we do not know whether a particular resume should be hired or not, but most would agree that the answer should not depend on the candidate's name.
-  </p><p>
-    For biases affecting probabilistic reasoning and decision making, there is a long literature exploring how people decide between gambles such as "Would you prefer <span>$2</span> with certainty or <span>$1</span> 40% of the time and <span>$3</span> otherwise?"<d-cite key="kahneman1979prospect,tversky1992advances"/>. For example, Erev et al. constructed an 11-dimensional space of gambles sufficient to reproduce 14 known cognitive biases, from which new instances can be algorithmically generated<d-cite key="erev2017anomalies"/>.  Would debates about gambles reduce cognitive biases?  One difficulty here is that simple gambles might fail the "definitive argument longer than debate limit" criteria if an expected utility calculation is sufficient to prove the answer, making it difficult for a lying debater to meaningfully compete.
-  </p><p>
-    Interestingly, Chen et al. used a similar setup to human+human+human debate to improve the quality of human data collected in a synthetic "Relation Extraction" task<d-cite key="chen2018cicero"/>.  People were first asked for direct answers, then pairs of people who disagreed were asked to discuss and possibly update their answers.  Here the debaters and judges are the same, but the overall goal of extracting higher quality information from humans is shared with debate.
-  </p>
-
-  <h2 id="questions">Questions social science can help us answer</h2>
-  <p>
-    We've laid out the general program for learning AI goals by asking humans questions, and discussed how to use debate to strengthen what we can learn by targeting the reasoning behind conclusions.  Whether we use direct questions or something like debate, any intervention that gives us higher quality answers is more likely to produce aligned AI.  The quality of those answers depends on the human judges, and social science research can help to measure answer quality and improve it.  Let's go into more detail about what types of questions we want to answer, and what we hope to do with that information.  Although we will frame these questions as they apply to debate, most of them apply to any other method which learns goals from humans.
-  </p>
-  <ol>
-    <li><strong>How skilled are people as judges by default?</strong>
-    If we ran debate using a person chosen at random as the judge, and gave them no training, would the result be aligned?  A person picked at random might be vulnerable to convincing fallacious reasoning<d-cite key="hahn2007rationality"/>, leading AI to employ such reasoning. Note that the debaters are not chosen at random: once the judge is fixed, we care about debaters who either learn to help the judge (in the good case) or to exploit the judge's weaknesses (in the bad case).</li>
-    <li><strong>Can we distinguish good judges from bad judges?</strong>
-    People likely differ in the ability to judge debates.  There are many filters we could use to identify good judges: comparing their verdicts to those of other judges, to people given more time to think, or to known expert judgment<d-footnote>Note that domain expertise may be quite different from what makes a good judge of debate.  Although there is evidence that domain expertise reduces bias<d-cite key="bornstein2001rationality"/>, "expert" political forecasters may actually be worse than non-experts (<d-cite key="tetlock2017expert"/>, chapter 3).</d-footnote>.  Ideally we would like filters that do not require an independent source of truth, though at experiment time we will need a source of truth to know whether a filter works.  It is not obvious a priori that good filters exist, and any filter would need careful scrutiny to ensure it does not introduce bias into our choice of judges.</li>
-    <li><strong>Does judge ability generalize across domains?</strong>
-    If judge ability in one domain fails to transfer to other domains, we will have low confidence that it transfers to new questions and arguments arising from highly capable AI debaters.  This generalization is necessary to trust debate as a method for alignment, especially once we move to questions where no independent source of truth is available.  We emphasize that judge ability is not the same as knowledge: there is evidence that expertise often fails to generalize across domains<d-cite key="chi2006two"/>, but argument evaluation could transfer where expertise does not.</li>
-    <li><strong>Can we train people to be better judges?</strong>
-    Peer review, practice, debiasing<d-cite key="larrick2004debiasing"/>, formal training such as argument mapping<d-cite key="dwyer2012evaluation"/>, expert panels, tournaments<d-cite key="tetlock2014forecasting"/>, and other interventions may make people better at judging debates.  Which mechanisms work best?</li>
-    <li><strong>What questions are people better at answering?</strong>
-    If we know that humans are bad at answering certain types of questions, we can switch to reliable formulations.  For example, phrasing questions in frequentist terms may reduce known cognitive biases<span class="nowrap"><d-cite key="gigerenzer1991make"/>.</span>  Graham et al. argue that different political views follow from different weights placed on fundamental moral considerations, and similar analysis could help understand where we can expect moral disagreements to persist after reflective equilibrium<d-cite key="graham2009liberals"/>.  In cases where reliable answers are unavailable, we need to ensure that trained models know their own limits, and express uncertainty or disagreement as required.</li>
-    <li><strong>Are there ways to restrict debate to make it easier to judge?</strong>
-    People might be better at judging debates formulated in terms of calm, factual statements, and worse at judging debates designed to trigger strong emotions.  Or, counterintuitively, it could be the other way around<d-cite key="goel2011negative"/>.  If we know which styles of debates that people are
-    better at judging, we may be able to restrict AI debaters to these styles.</li>
-    <li><strong>How can people work together to improve quality?</strong>
-    If individuals are insufficient judges, are teams of judges better?  Majority vote is the simplest option, but perhaps several people talking through an answer together is stronger, either actively or after the fact through peer review.  Condorcet's jury theorem implies that majority votes can amplify weakly good judgments to strong judgments (or weakly bad judgments to worse)<d-cite key="list2001epistemic"/>, but aggregation may be more complex in cases of probabilistic judgment<d-cite key="list2002aggregating"/>.  Teams could be informal or structured; see the Delphi technique for an example of structured teams applied to forecasting<d-cite key="rowe1999delphi"/>.</li>
-  </ol>
-  <p>
-    We believe these questions require social science experiments to satisfactorily answer.
-  </p><p>
-    Given our lack of experience outside of ML, we are not able to precisely articulate all of the different experiments we need.  The only way to fix this is to talk to more people with different backgrounds and expertise.  We have started this process, but are eager for more conversations with social scientists about what experiments could be run, and encourage other AI safety efforts to engage similarly.
-  </p>
-
-  <h2 id="optimism">Reasons for optimism</h2>
-  <p>
-    We believe that understanding how humans interact with long-term AI alignment is difficult but possible.  However, this would be a new research area, and we want to be upfront about the uncertainties involved.  In this section and the next, we discuss some reasons for optimism and pessimism about whether this research will succeed.  We focus on issues specific to human uncertainty and associated social science research; for similar discussion on ML uncertainty in the case of debate we refer to our previous work<d-cite key="irving2018debate"/>.
-  </p>
-  <h3 id="engineering">Engineering vs. science</h3>
-  <p>
-    Most social science seeks to understand humans "in the wild": results that generalize to people going about their everyday lives.  With limited control over these lives, differences between laboratory and real life are bad from the scientific perspective.  In contrast, AI alignment seeks to extract the best version of what humans want: our goal is engineering rather than science, and we have more freedom to intervene.  If judges in debate need training to perform well, we can provide that training.  If some people still do not provide good data, we can remove them from experiments (as long as this filter does not create too much bias).  This freedom to intervene means that some of the difficulty in understanding and improving human reasoning may not apply.  However, science is still required: once our interventions are in place, we need to correctly know whether our methods work.  Since our experiments will be an imperfect model of the final goal, careful design will be necessary to minimize this mismatch, just as is required by existing social science.
-  </p>
-  <h3 id="no-need-to-answer-all-questions">We don't need to answer all questions</h3>
-  <p>
-    Our most powerful intervention is to give up: to recognize that we are unable to answer some types of questions, and instead prevent AI systems from pretending to answer.  Humans might be good judges on some topics but not others, or with some types of reasoning but not others; if we discover that we can adjust our goals appropriately.  Giving up on some types of questions is achievable either on the ML side, using careful uncertainty modeling to know when we do not know, or on the human side by training judges to understand their own areas of uncertainty.  Although we will attempt to formulate ML systems that automatically detect areas of uncertainty, any information we can gain on the social science side about human uncertainty can be used both to augment ML uncertainty modeling and to test whether ML uncertainty modeling works.
-  </p>
-  <h3 id="relative-accuracy-may-be-enough">Relative accuracy may be enough</h3>
-  <p>
-    Say we have a variety of different ways to structure debate with humans.  Ideally, we would like to achieve results of the form "debate structure $A$ is truth-seeking with 90% confidence".  Unfortunately, we may be unconfident that an absolute result of this form will generalize to advanced AI systems: it may hold for an experiment with simple tasks but break down later on.  However, even if we can't achieve such absolute results, we can still hope for relative results of the form "debate structure $A$ is reliably better than debate structure $B$".  Such a result may be more likely to generalize into the future, and assuming it does we will know to use structure $A$ rather than $B$.
-  </p>
-  <h3 id="no-need-to-pin-down-best-alignment-scheme">We don't need to pin down the best alignment scheme</h3>
-  <p>
-    As the AI safety field progresses to increasingly advanced ML systems, we expect research on the ML side and the human side to merge.  Starting social science experiments prior to this merging will give the field a head start, but we can also take advantage of the expected merging to make our goals easier.  If social science research narrows the design space of human-friendly AI alignment algorithms but does not produce a single best scheme, we can test the smaller design space once the machines are ready.
-  </p>
-  <h3 id="negative-result-would-be-important">A negative result would be important!</h3>
-  <p>
-    If we test an AI alignment scheme from the social science perspective and it fails, we've learned valuable information.  There are a variety of proposed alignment schemes, and learning which don't work early gives us more time to switch to others, or to intervene on a policy level to slow down dangerous development.  In fact, given our belief that AI alignment is harder for more advanced agents, a negative result might be easier to believe and thus more valuable that a less trustworthy positive result.
-  </p>
-
-  <h2 id="worry">Reasons to worry</h2>
-  <p>
-    We turn next to reasons social science experiments about AI alignment might fail to produce useful results.  We emphasize that useful results might be both positive and negative, so these are not reasons why alignment schemes might fail.  Our primary worry is one sided, that experiments would say an alignment scheme works when in fact it does not, though errors in the other direction are also undesirable.
-  </p>
-  <h3 id="conflicting-desiderate">Our desiderata are conflicting</h3>
-  <p>
-    As mentioned before, some of our criteria when picking experimental tasks are in conflict.  We want tasks that are sufficiently interesting (not too easy), with a source of verifiable ground truth, are not too hard, etc.  "Not too easy" and "not too hard" are in obvious conflict, but there are other more subtle difficulties.  Domain experts with the knowledge to debate interesting tasks may not be the same people capable of lying effectively, and both restrictions make it hard to gather large volumes of data.  Lying effectively is required for a meaningful experiment, since a trained AI may have no trouble lying unless lying is a poor strategy to win debates.  Experiments to test whether ethical biases interfere with judgment may make it more difficult to find tasks with reliable ground truth, especially on subjects with significant disagreement across people.  The natural way out is to use many different experiments to cover different aspects of our uncertainty, but this would take more time and might fail to notice interactions between desiderata.
-  </p>
-  <h3 id="measure-judge-quality">We want to measure judge quality given optimal debaters</h3>
-  <p>
-    For debate, our end goal is to understand if the judge is capable of determining who is telling the truth.  However, we specifically care whether the judge performs well given that the debaters are performing well.  Thus our experiments have an inner/outer optimization structure: we first train the debaters to debate well, then measure how well the judges perform.  This increases time and cost: if we change the task, we may need to find new debaters or retrain existing debaters.  Worse, the human debaters may be bad at performing the task, either out of inclination or ability.  Poor performance is particularly bad if it is one sided and applies only to lying: a debater might be worse at lying out of inclination or lack of practice, and thus a win for the honest debater might be misleading.
-  </p>
-  <h3 id="ml-algorithms-will-change">ML algorithms will change</h3>
-  <p>
-    It is unclear when or if ML systems will reach various levels of capability, and the algorithms used to train them will evolve over time.  The AI alignment algorithms of the future may be similar to the proposed algorithms of today, or they may be very different.  However, we believe that knowledge gained on the human side will partially transfer: results about debate will teach us about how to gather data from humans even if debate is superseded.  The algorithms may change; humans will not.
-  </p>
-  <h3 id="strong-out-of-domain-generalization">Need strong out-of-domain generalization</h3>
-  <p>
-    Regardless of how carefully designed our experiments are, human+human+human debate will not be a perfect match to AI+AI+human debate.  We are seeking research results that generalize to the setting where we replace the human debaters (or similar) with AIs of the future, which is a hard ask.  This problem is fundamental: we do not have the advanced AI systems of the future to play with, and want to learn about human uncertainty starting now.
-  </p>
-  <h3 id="lack-of-philosophical-clarity">Lack of philosophical clarity</h3>
-  <p>
-    Any AI alignment scheme will be both an algorithm for training ML systems and a proposed definition of what it means to be aligned.  However, we do not expect humans to conform to any philosophically consistent notion of values, and concepts like reflective equilibrium must be treated with caution in case they break down when applied to real human judgement.  Fortunately, algorithms like debate need not presuppose philosophical consistency: a back and forth conversation to convince a human judge makes sense even if the human is leaning on heuristics, intuition, and emotion.  It is not obvious that debate works in this messy setting, but there is hope if we take advantage of inaction bias, uncertainty modeling, and other escape hatches.  We believe lack of philosophical clarity is an argument for investing in social science research: if humans are not simple, we must engage with their complexity.
-  </p>
+  <p>Details on the methods used in the project</p>
 
-  <h2 id="scale">The scale of the challenge</h2>
-  <p>
-    Long-term AI safety is particularly important if we develop artificial general intelligence (AGI), which the OpenAI Charter defines as highly autonomous systems that outperform humans at most economically valuable work<d-cite key="openai2018charter"/>.  If we want to train an AGI with reward learning from humans, it is unclear how many samples will be required to align it.  As much as possible, we can try to replace human samples with knowledge about the world gained by reading language, the internet, and other sources of information.  But it is likely that a fairly large number of samples from people will still be required.  Since more samples means less noise and more safety, if we are uncertain about how many samples we need then we will want a lot of samples.
-  </p><p>
-    A lot of samples would mean recruiting a lot of people.  We cannot rule out needing to involve thousands to tens of thousands of people for millions to tens of millions of short interactions: answering questions, judging debates, etc.  We may need to train these people to be better judges, arrange for peers to judge each other's reasoning, determine who is doing better at judging and give them more weight or a more supervisory role, and so on.  Many researchers would be required on the social science side to extract the highest quality information from the judges.
-  </p><p>
-    A task of this scale would be a large interdisciplinary project, requiring close collaborations in which people of different backgrounds fill in each other's missing knowledge.  If machine learning reaches this scale, it is important to get a head start on the collaborations soon.
-  </p>
+  <h2 id="conclusion">Conclusion: How did the project create impact?</h2>
 
-  <h2 id="conclusion">Conclusion: how you can help</h2>
-  <p>
-    We have argued that the AI safety community needs social scientists to tackle a major source of uncertainty about AI alignment algorithms: will humans give good answers to questions?  This uncertainty is difficult to tackle with conventional machine learning experiments, since machine learning is primitive.  We are still in the early days of performance on natural language and other tasks, and problems with human reward learning may only show up on tasks we cannot yet tackle.
-  </p><p>
-    Our proposed solution is to replace machine learning with people, at least until ML systems can participate in the complexity of debates we are interested in.  If we want to understand a game played with ML and human participants, we replace the ML participants with people, and see how the all human game plays out.  For the specific example of debate, we start with debates with two ML debaters and a human judge, then switch to two human debaters and a human judge.  The result is a pure human experiment, motivated by machine learning but available to anyone with a solid background in experimental social science.  It won't be an easy experiment, which is all the more reason to start soon.
-  </p><p>
-    If you are a social scientist interested in these questions, please talk to AI safety researchers!  We are interested in both conversation and close collaboration.  There are many institutions engaged with safety work using reward learning, including our own institution <a href="https://openai.com">OpenAI</a>, <a href="https://deepmind.com">DeepMind</a>, and <a href="https://humancompatible.ai">Berkeley's CHAI</a>.  The AI safety organization <a href="https://ought.org">Ought</a> is already exploring similar questions, asking how iterated amplification behaves with humans.
-  </p><p>
-    If you are a machine learning researcher interested in or already working on safety, please think about how alignment algorithms will work once we advance to tasks beyond the abilities of current machine learning.  If your preferred alignment scheme uses humans in an important way, can you simulate the future by replacing some or all ML components with people?  If you can imagine these experiments but don't feel you have the expertise to perform them, find someone who does.
-  </p>
+  <p>Conclusion on the project contributions</p>
 
 </d-article>
 
 <d-appendix>
   <h3 id="acknowledgements">Acknowledgments</h3>
   <p>
-    We are grateful to Gillian Hadfield, Dario Amodei, Brian Christian, Michael Page, David Manley, Josh Kalla, Remco Zwetsloot, Baobao Zhang, David Moss, Daniel Greene, Daniel Ziegler, Danny Hernandez, Mahendra Prasad, Liv Boeree, Igor Kurganov, Cate Hall, Ashley Pilipiszyn, and others for extensive feedback on the article.  We had conversations with many social scientists during the process of writing this article, including Mariano-Florentino Cuéllar, Philip Tetlock, Rob MacCoun, John Ahlquist, the participants of a Stanford CASBS workshop organized by Margaret Levi and Federica Carugati, Tom Griffiths, Elizabeth Rhodes, Alex Newar, and Stefan Schubert. We are also grateful to participants at the EA Global 2018: London conference where this work was presented.  Paul Christiano participated in the domain expert debates and extensive discussion.  We emphasize that all mistakes in content and terminology are our own, not those of the acknowledged.
-  </p><p>
-    On the Distill side, we are grateful to Arvind Satyanarayan for handling the review process, and Shan Carter and Ludwig Schubert for extensive help on structure, diagrams, and formatting.
-  </p><p>
-    The debate tree diagram was made with the help of Mike Bostock's <a href="https://beta.observablehq.com/@mbostock/tree-o-matic">tree-o-matic notebook</a>.
+    We are grateful to x, y, z, and others for extensive feedback on the article.
   </p>
 
-  <h2 id="quantum">Example debate: Quantum SAT solver</h2>
-  <p>
-    We've conducted a few informal <a href="#expert">domain expert debates</a> among people at OpenAI.  Here is one example on the following question:
-  </p><p>
-    <strong>Question:</strong> How fast can quantum computers solve boolean satisfiability?
-  </p><p>
-    Note that this question does not have a proven answer, so "correctness" is a mix of mathematical knowledge and subjective human judgement.  Both debaters knew quantum computation and had the same beliefs about the question, but one was trying to lie.  The judge was invited to comment and ask questions throughout the debate.  Full interaction from the judge was a change from a previous debate, where lack of judge participation meant that misinterpretations of statements early in the debate were not corrected.  Once the transcript was complete, it was shown to several other people to ensure that the first judge's confidence wasn't purely a result of facial expressions in the room or other side channels (this is far from perfect given the judge interaction).  All judges correctly determined who was telling the truth.
-  </p><p>
-    Each debater was given a budget of one sentence from Wikipedia to truthfully cite, excluding pages directly discussing the difficulty of quantum SAT solvers.  All their other statements could be lies.  The debaters were not allowed to take back moves, mostly to save time: a previous debate trying to get "the best transcript" by allowing debaters to rewind the game took many hours to complete.  As a result, the original debate transcript goes off on a variety of tangents before settling on a central disagreement.  As an illustration of how long the debate "might have been" if such tangents were avoided, we have provided a pruned transcript which culls away paths not relevant to the final deciding "line of argument".
-  </p><p>
-    Neither the original transcript or the pruned version are intended to be taken as real data: they serve only as an example of the types of interaction one might find in debate with domain experts and a lay judge.
-  </p>
-
-  <h3 id="instructions-for-judges">Instructions for judges</h3>
-  <ol class="instructions">
-    <li>The question of the debate and answers are given below.</li>
-    <li>Blue and Red are domain expert debaters.</li>
-    <li>The Judge is a non-domain expert that interacts with the debaters to help them model judges.</li>
-    <li>The debate was capped at 3000 words (for the unpruned version).</li>
-    <li>Check the time: we don't want judging to take more than 15 to 20 minutes.</li>
-    <li>Read the question and Blue and Red's answers and before reading the debate answer question (2).</li>
-    <li>Read through the debate transcript.</li>
-    <li>Answer the questions about the debate. Your key goal is to determine who is telling the truth.</li>
-  </ol>
-
-  <h3 id="questions-for-judges">Questions for judges</h3>
-  <ol class="instructions">
-    <li>Who do you think is telling the truth: Blue or Red?</li>
-    <li>How confident were you about whose claim was true before you read the debate?</li>
-    <li>How confident were you about whose claim was true at the end of the debate?</li>
-    <li>How useful did you find the interactive judge in this debate?</li>
-    <li>How did this compare with the previous transcript you were shown?</li>
-    <li>Do you have any other comments about the debate?</li>
-  </ol>
-
-  <h3 id="debate-transcript">Transcript</h3>
-  <div id="debate-transcripts">
-    <input class="tab" id="tab-debate-pruned" type="radio" name="tabs" checked>
-    <label for="tab-debate-pruned">Pruned</label>
-    <input class="tab"  id="tab-debate-unpruned" type="radio" name="tabs">
-    <label for="tab-debate-unpruned">Full length</label>
-    <section class="debate" id="debate-pruned">
-      <debate name="pruned"/>
-      <label for="tab-debate-unpruned" onclick="document.getElementById('debate-transcript').scrollIntoView({behavior: 'smooth', block: 'start'})">Read full length transcript</label>
-    </section>
-    <section class="debate" id="debate-unpruned">
-      <debate name="unpruned"/>
-    </section>
-  </div>
-
   <d-footnote-list></d-footnote-list>
   <d-bibliography src="references.bib"></d-bibliography>
   <d-citation-list></d-citation-list>
diff --git a/src/util.js b/src/util.js
index 7020b50..1307037 100644
--- a/src/util.js
+++ b/src/util.js
@@ -1,3 +1,3 @@
 export function range(n){
   return Array(n).fill().map((_, i) => i);
-}
\ No newline at end of file
+}
diff --git a/static/references.bib b/static/references.bib
index 9aeea94..72f5e35 100644
--- a/static/references.bib
+++ b/static/references.bib
@@ -1,658 +1,10 @@
-@article{irving2018debate,
-  title={AI safety via debate},
-  author={Irving, Geoffrey and Christiano, Paul and Amodei, Dario},
-  journal={arXiv preprint arXiv:1805.00899},
-  year={2018},
-  url={https://arxiv.org/abs/1805.00899},
-}
-
-@misc{irving2018debateblog,
-  author={Irving, Geoffrey and Amodei, Dario},
-  title={AI safety via debate},
-  type={Blog},
-  number={May 3},
-  year={2018},
-  url={https://blog.openai.com/debate},
-}
-
-@article{silver2017alphazero,
-  title={Mastering chess and shogi by self-play with a general reinforcement learning algorithm},
-  author={Silver, David and Hubert, Thomas and Schrittwieser, Julian and Antonoglou, Ioannis and Lai, Matthew and Guez, Arthur and Lanctot, Marc and Sifre, Laurent and Kumaran, Dharshan and Graepel, Thore and others},
-  journal={arXiv preprint arXiv:1712.01815},
-  year={2017},
-  url={https://arxiv.org/abs/1712.01815},
-}
-
-@article{campbell2002deepblue,
-  title = "Deep Blue",
-  journal = "Artificial Intelligence",
-  volume = "134",
-  number = "1",
-  pages = "57 - 83",
-  year = "2002",
-  issn = "0004-3702",
-  url = "https://doi.org/10.1016/S0004-3702(01)00129-1",
-  author = "Murray Campbell and A.Joseph Hoane and Feng-hsiung Hsu",
-  keywords = "Computer chess, Game tree search, Parallel search, Selective search, Search extensions, Evaluation function",
-  abstract = "Deep Blue is the chess machine that defeated then-reigning World Chess Champion Garry Kasparov in a six-game match in 1997. There were a number of factors that contributed to this success, including: •a single-chip chess search engine,•a massively parallel system with multiple levels of parallelism,•a strong emphasis on search extensions,•a complex evaluation function, and•effective use of a Grandmaster game database. This paper describes the Deep Blue system, and gives some of the rationale that went into the design decisions behind Deep Blue."
-}
-
-@inproceedings{mishra2017simple,
-  title={A simple neural attentive meta-learner},
-  author={Mishra, Nikhil and Rohaninejad, Mostafa and Chen, Xi and Abbeel, Pieter},
-  booktitle={NIPS 2017 Workshop on Meta-Learning},
-  year={2017},
-  url={https://arxiv.org/abs/1707.03141},
-}
-
-@article{chi2006two,
-  title={Two approaches to the study of experts’ characteristics},
-  author={Chi, Michelene T. H.},
-  journal={The Cambridge Handbook of Expertise and Expert Performance},
-  pages={21--30},
-  year={2006},
-  url={https://learnlab.org/uploads/mypslc/publications/chi%20two%20approaches%20chapter%202006.pdf}
-}
-
-@article{dwyer2012evaluation,
-  title={An evaluation of argument mapping as a method of enhancing critical thinking performance in e-learning environments},
-  author={Dwyer, Christopher P and Hogan, Michael J and Stewart, Ian},
-  journal={Metacognition and Learning},
-  volume={7},
-  number={3},
-  pages={219--244},
-  year={2012},
-  publisher={Springer},
-  url={https://link.springer.com/article/10.1007/s11409-012-9092-1}
-}
-
-@article{falk2018persuasion,
-  title={Persuasion, influence, and value: Perspectives from communication and social neuroscience},
-  author={Falk, Emily and Scholz, Christin},
-  journal={Annual review of psychology},
-  volume={69},
-  year={2018},
-  url={https://doi.org/10.1146/annurev-psych-122216-011821}
-}
-
-@article{flynn2017nature,
-  title={The nature and origins of misperceptions: Understanding false and unsupported beliefs about politics},
-  author={Flynn, DJ and Nyhan, Brendan and Reifler, Jason},
-  journal={Political Psychology},
-  volume={38},
-  pages={127--150},
-  year={2017},
-  publisher={Wiley Online Library},
-  url={https://doi.org/10.1111/pops.12394}
-}
-
-@article{hahn2007rationality,
-  title={The rationality of informal argumentation: A Bayesian approach to reasoning fallacies},
-  author={Hahn, Ulrike and Oaksford, Mike},
-  journal={Psychological review},
-  volume={114},
-  number={3},
-  pages={704},
-  year={2007},
-  publisher={American Psychological Association},
-  url={https://doi.org/10.1037/0033-295X.114.3.704}
-}
-
-@article{haidt2000moral,
-  title={Moral dumbfounding: When intuition finds no reason},
-  author={Haidt, Jonathan and Bjorklund, Fredrik and Murphy, Scott},
-  journal={Unpublished manuscript, University of Virginia},
-  year={2000}
-}
-
-@article{hewstone2002intergroup,
-  title={Intergroup bias},
-  author={Hewstone, Miles and Rubin, Mark and Willis, Hazel},
-  journal={Annual Review of Psychology},
-  volume={53},
-  number={1},
-  pages={575--604},
-  year={2002},
-  publisher={Annual Reviews 4139 El Camino Way, PO Box 10139, Palo Alto, CA 94303-0139, USA},
-  url={https://doi.org/10.1146/annurev.psych.53.100901.135109}
-}
-
-@article{gigerenzer1991make,
-  title={How to make cognitive illusions disappear: Beyond "heuristics and biases"},
-  author={Gigerenzer, Gerd},
-  journal={European review of social psychology},
-  volume={2},
-  number={1},
-  pages={83--115},
-  year={1991},
-  publisher={Taylor & Francis},
-  url={https://doi.org/10.1080/14792779143000033}
-}
-
-@article{graham2009liberals,
-  title={Liberals and conservatives rely on different sets of moral foundations},
-  author={Graham, Jesse and Haidt, Jonathan and Nosek, Brian A},
-  journal={Journal of personality and social psychology},
-  volume={96},
-  number={5},
-  pages={1029},
-  year={2009},
-  publisher={American Psychological Association},
-  url={https://www-bcf.usc.edu/~jessegra/papers/GrahamHaidtNosek.2009.Moral%20foundations%20of%20liberals%20and%20conservatives.JPSP.pdf}
-}
-
-@article{goel2011negative,
-  title={Negative emotions can attenuate the influence of beliefs on logical reasoning},
-  author={Goel, Vinod and Vartanian, Oshin},
-  journal={Cognition and Emotion},
-  volume={25},
-  number={1},
-  pages={121--131},
-  year={2011},
-  publisher={Taylor & Francis},
-  url={https://doi.org/10.1080/02699931003593942}
-}
-
-@article{greene2002and,
-  title={How (and where) does moral judgment work?},
-  author={Greene, Joshua and Haidt, Jonathan},
-  journal={Trends in cognitive sciences},
-  volume={6},
-  number={12},
-  pages={517--523},
-  year={2002},
-  publisher={Elsevier},
-  url={https://static.squarespace.com/static/54763f79e4b0c4e55ffb000c/t/5477ccf2e4b07347e76a53c9/1417137394112/how-and-where-does-moral-judgment-work.pdf}
-}
-
-@book{goodman1983fact,
-  title={Fact, fiction, and forecast},
-  author={Goodman, Nelson},
-  year={1983},
-  publisher={Harvard University Press},
-}
-
-@article{larrick2004debiasing,
-  title={Debiasing},
-  author={Larrick, Richard P},
-  journal={Blackwell Handbook of Judgment and Decision Making},
-  pages={316--338},
-  year={2004},
-  publisher={Wiley Online Library},
-  url={https://doi.org/10.1002/9780470752937.ch16}
-}
-
-@article{list2001epistemic,
-  title={Epistemic democracy: Generalizing the Condorcet jury theorem},
-  author={List, Christian and Goodin, Robert E},
-  journal={Journal of political philosophy},
-  volume={9},
-  number={3},
-  pages={277--306},
-  year={2001},
-  publisher={Wiley Online Library},
-  url={https://doi.org/10.1111/1467-9760.00128},
-}
-
-@article{list2002aggregating,
-  title={Aggregating sets of judgments: An impossibility result},
-  author={List, Christian and Pettit, Philip},
-  journal={Economics \& Philosophy},
-  volume={18},
-  number={1},
-  pages={89--110},
-  year={2002},
-  publisher={Cambridge University Press},
-  url={https://www.princeton.edu/~ppettit/papers/Aggregating_EconomicsandPhilosophy_2002.pdf}
-}
-
-@article{paluck2016overcome,
-  title={How to overcome prejudice},
-  author={Paluck, Elizabeth Levy},
-  journal={Science},
-  volume={352},
-  number={6282},
-  pages={147--147},
-  year={2016},
-  publisher={American Association for the Advancement of Science},
-  url={http://www.betsylevypaluck.com/s/Paluck2016.pdf}
-}
-
-@article{petty2010attitude,
-  title={Attitude change},
-  author={Petty, Richard E and Brinol, Pablo},
-  journal={Advanced social psychology: The state of the science},
-  pages={217--259},
-  year={2010},
-  url={https://doi.org/10.1146/annurev.psych.48.1.609},
-}
-
-@book{rawls2009theory,
-  title={A theory of justice},
-  author={Rawls, John},
-  year={2009},
-  publisher={Harvard university press},
-  url={https://en.wikipedia.org/wiki/A_Theory_of_Justice}
-}
-
-@article{rowe1999delphi,
-  title = "The Delphi technique as a forecasting tool: issues and analysis",
-  journal = "International Journal of Forecasting",
-  volume = "15",
-  number = "4",
-  pages = "353 - 375",
-  year = "1999",
-  issn = "0169-2070",
-  doi = "https://doi.org/10.1016/S0169-2070(99)00018-7",
-  url = "http://www.sciencedirect.com/science/article/pii/S0169207099000187",
-  author = "Gene Rowe and George Wright",
-  keywords = "Delphi, Interacting groups, Statistical group, Structured groups",
-  abstract = "This paper systematically reviews empirical studies looking at the effectiveness of the Delphi technique, and provides a critique of this research. Findings suggest that Delphi groups outperform statistical groups (by 12 studies to two with two ‘ties’) and standard interacting groups (by five studies to one with two ‘ties’), although there is no consistent evidence that the technique outperforms other structured group procedures. However, important differences exist between the typical laboratory version of the technique and the original concept of Delphi, which make generalisations about ‘Delphi’ per se difficult. These differences derive from a lack of control of important group, task, and technique characteristics (such as the relative level of panellist expertise and the nature of feedback used). Indeed, there are theoretical and empirical reasons to believe that a Delphi conducted according to ‘ideal’ specifications might perform better than the standard laboratory interpretations. It is concluded that a different focus of research is required to answer questions on Delphi effectiveness, focusing on an analysis of the process of judgment change within nominal groups."
-}
-
-@book{tetlock2017expert,
-  title={Expert political judgment: How good is it? How can we know?},
-  author={Tetlock, Philip E},
-  year={2017},
-  publisher={Princeton University Press},
-  url={https://press.princeton.edu/titles/11152.html}
-}
-
-@article{tetlock2014forecasting,
-  title={Forecasting tournaments: Tools for increasing transparency and improving the quality of debate},
-  author={Tetlock, Philip E and Mellers, Barbara A and Rohrbaugh, Nick and Chen, Eva},
-  journal={Current Directions in Psychological Science},
-  volume={23},
-  number={4},
-  pages={290--295},
-  year={2014},
-  publisher={Sage Publications Sage CA: Los Angeles, CA},
-  url={https://doi.org/10.1177%2F0963721414534257}
-}
-
-@article{tversky1974judgment,
-  title={Judgment under uncertainty: heuristics and biases},
-  author={Tversky, Amos and Kahneman, Daniel},
-  journal={Science},
-  volume={185},
-  number={4157},
-  pages={1124--1131},
-  year={1974},
-  publisher={American association for the advancement of science},
-  url={https://doi.org/10.1126/science.185.4157.1124}
-}
-
-@article{bornstein2001rationality,
-  title={Rationality in medical decision making: a review of the literature on doctors’ decision-making biases},
-  author={Bornstein, Brian H and Emler, A Christine},
-  journal={Journal of evaluation in clinical practice},
-  volume={7},
-  number={2},
-  pages={97--107},
-  year={2001},
-  publisher={Wiley Online Library},
-  url={https://doi.org/10.1046/j.1365-2753.2001.00284.x}
-}
-
-@article{christiano2017human,
-  title={Deep reinforcement learning from human preferences},
-  author={Christiano, Paul F and Leike, Jan and Brown, Tom and Martic, Miljan and Legg, Shane and Amodei, Dario},
-  booktitle={Advances in Neural Information Processing Systems},
-  pages={4299--4307},
-  year={2017},
-  url={https://arxiv.org/abs/1706.03741}
-}
-
-@article{christiano2018amplification,
-  title={Supervising strong learners by amplifying weak experts},
-  author={Christiano, Paul and Shlegeris, Buck and Amodei, Dario},
-  journal={arXiv preprint arXiv:1810.08575},
-  year={2018},
-  url={https://arxiv.org/abs/1810.08575}
-}
-
-@article{leike2017gridworlds,
-  title={AI safety gridworlds},
-  author={Leike, Jan and Martic, Miljan and Krakovna, Victoria and Ortega, Pedro A and Everitt, Tom and Lefrancq, Andrew and Orseau, Laurent and Legg, Shane},
-  journal={arXiv preprint arXiv:1711.09883},
-  year={2017},
-  url={https://arxiv.org/abs/1711.09883}
-}
-
-@inproceedings{ibarz2018,
-  title={Reward learning from human preferences and demonstrations in Atari},
-  author={Ibarz, Borja and Leike, Jan and Pohlen, Tobias and Irving, Geoffrey and Legg, Shane and Amodei, Dario},
-  booktitle={Advances in Neural Information Processing Systems},
-  year={2018},
-  url={https://arxiv.org/abs/1811.06521}
-}
-
-@inproceedings{buolamwini2018gender,
-  title={Gender shades: Intersectional accuracy disparities in commercial gender classification},
-  author={Buolamwini, Joy and Gebru, Timnit},
-  booktitle={Conference on Fairness, Accountability and Transparency},
-  pages={77--91},
-  year={2018},
-  url={http://proceedings.mlr.press/v81/buolamwini18a.html}
-}
-
-@article{nisbett1977introspection,
-  title={Telling more than we can know: Verbal reports on mental processes},
-  author={Nisbett, Richard E and Wilson, Timothy D},
-  journal={Psychological review},
-  volume={84},
-  number={3},
-  pages={231},
-  year={1977},
-  publisher={American Psychological Association},
-  url={http://psycnet.apa.org/doi/10.1037/0033-295X.84.3.231}
-}
-
-@article{pronin2009introspection,
-  title={The introspection illusion},
-  author={Pronin, Emily},
-  journal={Advances in experimental social psychology},
-  volume={41},
-  pages={1--67},
-  year={2009},
-  publisher={Elsevier},
-  url={https://doi.org/10.1016/S0065-2601(08)00401-2},
-}
-
-@article{bahdanau2018learning,
-  title={Learning to understand goal specifications by modelling reward},
-  author={Bahdanau, Dzmitry and Hill, Felix and Leike, Jan and Hughes, Edward and Kohli, Pushmeet and Grefenstette, Edward},
-  journal={arXiv preprint arXiv:1806.01946},
-  year={2018},
-  url={https://arxiv.org/abs/1806.01946},
-}
-
-@article{biyik2018batch,
-  title={Batch active preference-based learning of reward functions},
-  author={Bıyık, Erdem and Sadigh, Dorsa},
-  journal={arXiv preprint arXiv:1810.04303},
-  year={2018},
-  url={https://arxiv.org/abs/1810.04303},
-}
-
-@article{sugden2015looking,
-  title={Looking for a psychology for the inner rational agent},
-  author={Sugden, Robert},
-  journal={Social Theory and Practice},
-  volume={41},
-  number={4},
-  pages={579--598},
-  year={2015},
-  url={https://ueaeprints.uea.ac.uk/54622/1/psychology_of_inner_agent_1506_01.pdf}
-}
-
-@misc{mitchell2018fairness,
-  author = {Mitchell, Shira and Shadlen, Jackie},
-  title = {Mirror Mirror: Reflections on Quantitative Fairness},
-  type = {Blog},
-  number = {February},
-  year = {2018},
-  url = {https://shiraamitchell.github.io/fairness}
-}
-
-@inproceedings{hadfield2017inverse,
-  title={Inverse reward design},
-  author={Hadfield-Menell, Dylan and Milli, Smitha and Abbeel, Pieter and Russell, Stuart J and Dragan, Anca},
-  booktitle={Advances in Neural Information Processing Systems},
-  pages={6765--6774},
-  year={2017},
-  url={https://arxiv.org/abs/1711.02827}
-}
-
-@inproceedings{hadfield2016cooperative,
-  title={Cooperative inverse reinforcement learning},
-  author={Hadfield-Menell, Dylan and Russell, Stuart J and Abbeel, Pieter and Dragan, Anca},
-  booktitle={Advances in neural information processing systems},
-  pages={3909--3917},
-  year={2016},
-  url={https://arxiv.org/abs/1606.03137}
-}
-
-@article{mellers2015identifying,
-  title={Identifying and cultivating superforecasters as a method of improving probabilistic predictions},
-  author={Mellers, Barbara and Stone, Eric and Murray, Terry and Minster, Angela and Rohrbaugh, Nick and
-    Bishop, Michael and Chen, Eva and Baker, Joshua and Hou, Yuan and Horowitz, Michael and
-    Ungar, Lyle and Tetlock, Philip},
-  journal={Perspectives on Psychological Science},
-  volume={10},
-  number={3},
-  pages={267--281},
-  year={2015},
-  publisher={SAGE Publications Sage CA: Los Angeles, CA},
-  url={https://doi.org/10.1177%2F1745691615577794}
-}
-
-@article{mellers2014psychological,
-  title={Psychological strategies for winning a geopolitical forecasting tournament},
-  author={Mellers, Barbara and Ungar, Lyle and Baron, Jonathan and Ramos, Jaime and Gurcay, Burcu and
-    Fincher, Katrina and Scott, Sydney E and Moore, Don and Atanasov, Pavel and Swift, Samuel A and
-    Murray, Terry and Stone, Eric and Tetlock, Philip},
-  journal={Psychological science},
-  volume={25},
-  number={5},
-  pages={1106--1115},
-  year={2014},
-  publisher={Sage Publications Sage CA: Los Angeles, CA},
-  url={https://doi.org/10.1177%2F0956797614524255}
-}
-
-@book{tetlock2016superforecasting,
-  title={Superforecasting: The art and science of prediction},
-  author={Tetlock, Philip E and Gardner, Dan},
-  year={2016},
-  publisher={Random House}
-}
-
-@book{schopenhauer2013art,
-  title={The art of being right},
-  author={Schopenhauer, Arthur},
-  year={1896},
-  url={https://en.wikisource.org/wiki/The_Art_of_Being_Right}
-}
-
-@article{bertrand2004emily,
-  title={Are Emily and Greg more employable than Lakisha and Jamal? A field experiment on labor market discrimination},
-  author={Bertrand, Marianne and Mullainathan, Sendhil},
-  journal={American economic review},
-  volume={94},
-  number={4},
-  pages={991--1013},
-  year={2004},
-  url={https://www.nber.org/papers/w9873.pdf}
-}
-
-@article{kahneman1979prospect,
-  title={Prospect theory: An analysis of decisions under risk},
-  author={Kahneman, Daniel},
-  journal={Econometrica},
-  volume={47},
-  pages={278},
-  year={1979},
-  url={https://doi.org/10.2307/1914185}
-}
-
-@article{tversky1992advances,
-  title={Advances in prospect theory: Cumulative representation of uncertainty},
-  author={Tversky, Amos and Kahneman, Daniel},
-  journal={Journal of Risk and uncertainty},
-  volume={5},
-  number={4},
-  pages={297--323},
-  year={1992},
-  publisher={Springer}
-}
-
-@article{erev2017anomalies,
-  title={From anomalies to forecasts: Toward a descriptive model of decisions under risk, under ambiguity, and from experience},
-  author={Erev, Ido and Ert, Eyal and Plonsky, Ori and Cohen, Doron and Cohen, Oded},
-  journal={Psychological review},
-  volume={124},
-  number={4},
-  pages={369},
-  year={2017},
-  publisher={American Psychological Association},
-  url={https://doi.org/10.1037/rev0000062},
-}
-
-@misc{openai2018charter,
-  author={OpenAI},
-  title={OpenAI Charter},
-  type={Blog},
-  number={April 9},
-  year={2018},
-  url={https://blog.openai.com/charter},
-}
-
-@misc{openai2018five,
-  author={OpenAI},
-  title={OpenAI Five},
-  type={Blog},
-  number={June 25},
-  url={https://blog.openai.com/openai-five},
-  year={2018}
-}
-
-@misc{ought2018factored,
-  title={Factored Cognition},
-  author={Andreas Stuhlmüller},
-  number={May},
-  year={2018},
-  url={https://ought.org/presentations/factored-cognition-2018-05},
-}
-
-@inproceedings{evans2016inconsistent,
-  title={Learning the Preferences of Ignorant, Inconsistent Agents},
-  author={Evans, Owain and Stuhlm{\"u}ller, Andreas and Goodman, Noah D},
-  booktitle={AAAI},
-  pages={323--329},
-  year={2016},
-  url={https://arxiv.org/abs/1512.05832}
-}
-
-@book{christian2011human,
-  title={The Most Human Human: What Talking with Computers Teaches Us About What It Means to Be Alive},
-  author={Christian, Brian},
-  isbn={9780385533072},
-  url={https://books.google.com/books?id=uo2DW4XC7GgC},
-  year={2011},
-  publisher={Knopf Doubleday Publishing Group}
-}
-
-@article{leike2018scalable,
-  title={Scalable agent alignment via reward modeling: a research direction},
-  author={Leike, Jan and Krueger, David and Everitt, Tom and Martic, Miljan and Maini, Vishal and Legg, Shane},
-  journal={arXiv preprint arXiv:1811.07871},
-  year={2018},
-  url={https://arxiv.org/abs/1811.07871}
-}
-
-@article{wallach2016computational,
-  title={Computational Social Science: Towards a collaborative future},
-  author={Wallach, Hanna},
-  journal={Computational Social Science},
-  pages={307},
-  year={2016},
-  publisher={Cambridge University Press},
-  url={https://dirichlet.net/pdf/wallach15computational.pdf}
-}
-
-@book{kahneman2011thinking,
-  title={Thinking, fast and slow},
-  author={Kahneman, Daniel and Egan, Patrick},
-  volume={1},
-  year={2011},
-  publisher={Farrar, Straus and Giroux New York},
-  url={http://dx.doi.org/10.1037/h0099210}
-}
-
-@misc{thaler2008nudge,
-  title={Nudge: Improving decisions about health, wealth, and happiness},
-  author={Thaler, Richard H},
-  year={2008},
-  publisher={Yale University Press New Haven \& London}
-}
-
-@article{milli2017interpretable,
-  title={Interpretable and pedagogical examples},
-  author={Milli, Smitha and Abbeel, Pieter and Mordatch, Igor},
-  journal={arXiv preprint arXiv:1711.00694},
-  year={2017},
-  url={https://arxiv.org/abs/1711.00694}
-}
-
-@inproceedings{laskey2017comparing,
-  title={Comparing human-centric and robot-centric sampling for robot deep learning from demonstrations},
-  author={Laskey, Michael and Chuck, Caleb and Lee, Jonathan and Mahler, Jeffrey and Krishnan, Sanjay and Jamieson, Kevin and Dragan, Anca and Goldberg, Ken},
-  booktitle={Robotics and Automation (ICRA), 2017 IEEE International Conference on},
-  pages={358--365},
-  year={2017},
-  organization={IEEE},
-  url={https://arxiv.org/abs/1610.00850}
-}
-
-@incollection{sep-moral-anti-realism,
-  author={Joyce, Richard},
-  title={Moral Anti-Realism},
-  booktitle={The Stanford Encyclopedia of Philosophy},
-  editor={Edward N. Zalta},
-  url={https://plato.stanford.edu/archives/win2016/entries/moral-anti-realism},
-  year={2016},
-  edition={Winter 2016},
-  publisher={Metaphysics Research Lab, Stanford University}
-}
-
-@article{radford2018language,
-  title={Improving language understanding by generative pre-training},
-  author={Radford, Alec and Narasimhan, Karthik and Salimans, Tim and Sutskever, Ilya},
-  url={https://s3-us-west-2.amazonaws.com/openai-assets/research-covers/language-unsupervised/language_understanding_paper.pdf},
-  year={2018}
-}
-
-@techreport{bicchieri2017deviant,
-  title={Deviant or Wrong? The Effects of Norm Information on the Efficacy of Punishment},
-  author={Bicchieri, Cristina and Dimant, Eugen and Xiao, Erte and others},
-  year={2017},
-  url={https://www.monash.edu/business/economics/research/publications/publications2/0718Deviantbicchieri.pdf}
-}
-
-@article{henrich2010weirdest,
-  title={The weirdest people in the world?},
-  author={Henrich, Joseph and Heine, Steven J and Norenzayan, Ara},
-  journal={Behavioral and brain sciences},
-  volume={33},
-  number={2-3},
-  pages={61--83},
-  year={2010},
-  publisher={Cambridge University Press},
-  url={https://www.ssoar.info/ssoar/bitstream/handle/document/42104/ssoar-2010-henrich_et_al-The_weirdest_people_in_the.pdf}
-}
-
-@article{chen2018cicero,
-  title={Cicero: Multi-Turn, Contextual Argumentation for Accurate Crowdsourcing},
-  author={Chen, Quanze and Bragg, Jonathan and Chilton, Lydia B and Weld, Daniel S},
-  journal={arXiv preprint arXiv:1810.10733},
-  year={2018},
-  url={https://arxiv.org/abs/1810.10733}
-}
-
-@inproceedings{kelley1983wizard,
-  author = {Kelley, John F.},
-  title = {An empirical methodology for writing user-friendly natural language computer applications},
-  booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
-  series = {CHI '83},
-  year = {1983},
-  isbn = {0-89791-121-0},
-  location = {Boston, Massachusetts, USA},
-  pages = {193--196},
-  numpages = {4},
-  url = {http://doi.acm.org/10.1145/800045.801609},
-  doi = {10.1145/800045.801609},
-  acmid = {801609},
-  publisher = {ACM},
-  address = {New York, NY, USA},
+@article{rumelhart1986learning,
+  title={Learning representations by back-propagating errors},
+  author={Rumelhart, David E and Hinton, Geoffrey E and Williams, Ronald J},
+  journal={nature},
+  volume={323},
+  number={6088},
+  pages={533--536},
+  year={1986},
+  publisher={Nature Publishing Group UK London}
 }