httparchive.py

#!/usr/bin/env python
# Copyright 2010 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""View and edit HTTP Archives.

To list all URLs in an archive:
  $ ./httparchive.py ls archive.wpr

To view the content of all URLs from example.com:
  $ ./httparchive.py cat --host example.com archive.wpr

To view the content of a particular URL:
  $ ./httparchive.py cat --host www.example.com --full_path /foo archive.wpr

To view the content of all URLs:
  $ ./httparchive.py cat archive.wpr

To edit a particular URL:
  $ ./httparchive.py edit --host www.example.com --full_path /foo archive.wpr

To print statistics of an archive:
  $ ./httparchive.py stats archive.wpr

To print statistics of a set of URLs:
  $ ./httparchive.py stats --host www.example.com archive.wpr

To merge multiple archives
  $ ./httparchive.py merge --merged_file new.wpr archive1.wpr archive2.wpr ...
"""

import calendar
import certutils
import cPickle
import difflib
import email.utils
import httplib
import httpzlib
import json
import logging
import optparse
import os
import StringIO
import subprocess
import sys
import tempfile
import time
import urlparse
from collections import defaultdict



def LogRunTime(fn):
  """Annotation which logs the run time of the function."""
  def wrapped(self, *args, **kwargs):
    start_time = time.time()
    try:
      return fn(self, *args, **kwargs)
    finally:
      run_time = (time.time() - start_time) * 1000.0
      logging.debug('%s: %dms', fn.__name__, run_time)
  return wrapped


class HttpArchiveException(Exception):
  """Base class for all exceptions in httparchive."""
  pass


class HttpArchive(dict):
  """Dict with ArchivedHttpRequest keys and ArchivedHttpResponse values.

  Attributes:
    responses_by_host: dict of {hostname, {request: response}}. This must remain
        in sync with the underlying dict of self. It is used as an optimization
        so that get_requests() doesn't have to linearly search all requests in
        the archive to find potential matches.
  """

  def __init__(self):  # pylint: disable=super-init-not-called
    self.responses_by_host = defaultdict(dict)

  def __setstate__(self, state):
    """Influence how to unpickle.

    Args:
      state: a dictionary for __dict__
    """
    self.__dict__.update(state)
    self.responses_by_host = defaultdict(dict)
    for request in self:
      self.responses_by_host[request.host][request] = self[request]

  def __getstate__(self):
    """Influence how to pickle.

    Returns:
      a dict to use for pickling
    """
    state = self.__dict__.copy()
    del state['responses_by_host']
    return state

  def __setitem__(self, key, value):
    super(HttpArchive, self).__setitem__(key, value)
    if hasattr(self, 'responses_by_host'):
      self.responses_by_host[key.host][key] = value

  def __delitem__(self, key):
    super(HttpArchive, self).__delitem__(key)
    del self.responses_by_host[key.host][key]

  def get(self, request, default=None):
    """Return the archived response for a given request.

    Does extra checking for handling some HTTP request headers.

    Args:
      request: instance of ArchivedHttpRequest
      default: default value to return if request is not found

    Returns:
      Instance of ArchivedHttpResponse or default if no matching
      response is found
    """
    if request in self:
      return self[request]
    return self.get_conditional_response(request, default)

  def get_conditional_response(self, request, default):
    """Get the response based on the conditional HTTP request headers.

    Args:
      request: an ArchivedHttpRequest representing the original request.
      default: default ArchivedHttpResponse
          original request with matched headers removed.

    Returns:
      an ArchivedHttpResponse with a status of 200, 302 (not modified), or
          412 (precondition failed)
    """
    response = default
    if request.is_conditional():
      stripped_request = request.create_request_without_conditions()
      if stripped_request in self:
        response = self[stripped_request]
        if response.status == 200:
          status = self.get_conditional_status(request, response)
          if status != 200:
            response = create_response(status)
    return response

  def get_conditional_status(self, request, response):
    status = 200
    last_modified = email.utils.parsedate(
        response.update_date(response.get_header('last-modified')))
    response_etag = response.get_header('etag')
    is_get_or_head = request.command.upper() in ('GET', 'HEAD')

    match_value = request.headers.get('if-match', None)
    if match_value:
      if self.is_etag_match(match_value, response_etag):
        status = 200
      else:
        status = 412  # precondition failed
    none_match_value = request.headers.get('if-none-match', None)
    if none_match_value:
      if self.is_etag_match(none_match_value, response_etag):
        status = 304
      elif is_get_or_head:
        status = 200
      else:
        status = 412
    if is_get_or_head and last_modified:
      for header in ('if-modified-since', 'if-unmodified-since'):
        date = email.utils.parsedate(request.headers.get(header, None))
        if date:
          if ((header == 'if-modified-since' and last_modified > date) or
              (header == 'if-unmodified-since' and last_modified < date)):
            if status != 412:
              status = 200
          else:
            status = 304  # not modified
    return status

  @staticmethod
  def is_etag_match(request_etag, response_etag):
    """Determines whether the entity tags of the request/response matches.

    Args:
      request_etag: the value string of the "if-(none)-match:"
                    portion of the request header
      response_etag: the etag value of the response

    Returns:
      True on match, False otherwise
    """
    response_etag = response_etag.strip('" ')
    for etag in request_etag.split(','):
      etag = etag.strip('" ')
      if etag in ('*', response_etag):
        return True
    return False

  def get_requests(self, command=None, host=None, full_path=None, is_ssl=None,
                   use_query=True):
    """Return a list of requests that match the given args."""
    if host:
      return [r for r in self.responses_by_host[host]
              if r.matches(command, None, full_path, is_ssl,
                           use_query=use_query)]
    else:
      return [r for r in self
              if r.matches(command, host, full_path, is_ssl,
                           use_query=use_query)]

  def ls(self, command=None, host=None, full_path=None):
    """List all URLs that match given params."""
    return ''.join(sorted(
        '%s\n' % r for r in self.get_requests(command, host, full_path)))

  def cat(self, command=None, host=None, full_path=None):
    """Print the contents of all URLs that match given params."""
    out = StringIO.StringIO()
    for request in self.get_requests(command, host, full_path):
      print >>out, str(request)
      print >>out, 'Untrimmed request headers:'
      for k in request.headers:
        print >>out, '    %s: %s' % (k, request.headers[k])
      if request.request_body:
        print >>out, request.request_body
      print >>out, '---- Response Info', '-' * 51
      response = self[request]
      chunk_lengths = [len(x) for x in response.response_data]
      print >>out, ('Status: %s\n'
                    'Reason: %s\n'
                    'Headers delay: %s\n'
                    'Untrimmed response headers:') % (
          response.status, response.reason, response.delays['headers'])
      for k, v in response.original_headers:
        print >>out, '    %s: %s' % (k, v)
      print >>out, ('Chunk count: %s\n'
                    'Chunk lengths: %s\n'
                    'Chunk delays: %s') % (
          len(chunk_lengths), chunk_lengths, response.delays['data'])
      body = response.get_data_as_text()
      print >>out, '---- Response Data', '-' * 51
      if body:
        print >>out, body
      else:
        print >>out, '[binary data]'
      print >>out, '=' * 70
    return out.getvalue()

  def stats(self, command=None, host=None, full_path=None):
    """Print stats about the archive for all URLs that match given params."""
    matching_requests = self.get_requests(command, host, full_path)
    if not matching_requests:
      print 'Failed to find any requests matching given command, host, path.'
      return

    out = StringIO.StringIO()
    stats = {
        'Total': len(matching_requests),
        'Domains': defaultdict(int),
        'HTTP_response_code': defaultdict(int),
        'content_type': defaultdict(int),
        'Documents': defaultdict(int),
        }

    for request in matching_requests:
      stats['Domains'][request.host] += 1
      stats['HTTP_response_code'][self[request].status] += 1

      content_type = self[request].get_header('content-type')
      # Remove content type options for readability and higher level groupings.
      str_content_type = str(content_type.split(';')[0]
                            if content_type else None)
      stats['content_type'][str_content_type] += 1

      #  Documents are the main URL requested and not a referenced resource.
      if str_content_type == 'text/html' and not 'referer' in request.headers:
        stats['Documents'][request.host] += 1

    print >>out, json.dumps(stats, indent=4)
    return out.getvalue()

  def merge(self, merged_archive=None, other_archives=None):
    """Merge multiple archives into merged_archive by 'chaining' resources,
    only resources that are not part of the accumlated archive are added"""
    if not other_archives:
      print 'No archives passed to merge'
      return

    # Note we already loaded 'replay_file'.
    print 'Loaded %d responses' % len(self)

    for archive in other_archives:
      if not os.path.exists(archive):
        print 'Error: Replay file "%s" does not exist' % archive
        return

      http_archive_other = HttpArchive.Load(archive)
      print 'Loaded %d responses from %s' % (len(http_archive_other), archive)
      for r in http_archive_other:
        # Only resources that are not already part of the current archive
        # get added.
        if r not in self:
          print '\t %s ' % r
          self[r] = http_archive_other[r]
    self.Persist('%s' % merged_archive)

  def edit(self, command=None, host=None, full_path=None):
    """Edits the single request which matches given params."""
    editor = os.getenv('EDITOR')
    if not editor:
      print 'You must set the EDITOR environmental variable.'
      return

    matching_requests = self.get_requests(command, host, full_path)
    if not matching_requests:
      print ('Failed to find any requests matching given command, host, '
             'full_path.')
      return

    if len(matching_requests) > 1:
      print 'Found multiple matching requests. Please refine.'
      print self.ls(command, host, full_path)

    response = self[matching_requests[0]]
    tmp_file = tempfile.NamedTemporaryFile(delete=False)
    tmp_file.write(response.get_response_as_text())
    tmp_file.close()
    subprocess.check_call([editor, tmp_file.name])
    response.set_response_from_text(''.join(open(tmp_file.name).readlines()))
    os.remove(tmp_file.name)

  def find_closest_request(self, request, use_path=False):
    """Find the closest matching request in the archive to the given request.

    Args:
      request: an ArchivedHttpRequest
      use_path: If True, closest matching request's path component must match.
        (Note: this refers to the 'path' component within the URL, not the
         'full path' which includes the query string component.)

        If use_path=True, candidate will NOT match in example below
        e.g. request   = GET www.test.com/a?p=1
             candidate = GET www.test.com/b?p=1

        Even if use_path=False, urls with same paths are always favored.
        For example, candidate1 is considered a better match than candidate2.
          request    = GET www.test.com/a?p=1&q=2&r=3
          candidate1 = GET www.test.com/a?s=4
          candidate2 = GET www.test.com/b?p=1&q=2&r=3

    Returns:
      If a close match is found, return the instance of ArchivedHttpRequest.
      Otherwise, return None.
    """
    # Start with strictest constraints. This trims search space considerably.
    requests = self.get_requests(request.command, request.host,
                                 request.full_path, is_ssl=request.is_ssl,
                                 use_query=True)
    # Relax constraint: use_query if there is no match.
    if not requests:
      requests = self.get_requests(request.command, request.host,
                                   request.full_path, is_ssl=request.is_ssl,
                                   use_query=False)
    # Relax constraint: full_path if there is no match and use_path=False.
    if not requests and not use_path:
      requests = self.get_requests(request.command, request.host,
                                   None, is_ssl=request.is_ssl,
                                   use_query=False)

    if not requests:
      return None

    if len(requests) == 1:
      return requests[0]

    matcher = difflib.SequenceMatcher(b=request.cmp_seq)

    # quick_ratio() is cheap to compute, but ratio() is expensive. So we call
    # quick_ratio() on all requests, sort them descending, and then loop through
    # until we find a candidate whose ratio() is >= the next quick_ratio().
    # This works because quick_ratio() is guaranteed to be an upper bound on
    # ratio().
    candidates = []
    for candidate in requests:
      matcher.set_seq1(candidate.cmp_seq)
      candidates.append((matcher.quick_ratio(), candidate))

    candidates.sort(reverse=True, key=lambda c: c[0])

    best_match = (0, None)
    for i in xrange(len(candidates)):
      matcher.set_seq1(candidates[i][1].cmp_seq)
      best_match = max(best_match, (matcher.ratio(), candidates[i][1]))
      if i + 1 < len(candidates) and best_match[0] >= candidates[i+1][0]:
        break
    return best_match[1]

  def diff(self, request):
    """Diff the given request to the closest matching request in the archive.

    Args:
      request: an ArchivedHttpRequest
    Returns:
      If a close match is found, return a textual diff between the requests.
      Otherwise, return None.
    """
    request_lines = request.formatted_request.split('\n')
    closest_request = self.find_closest_request(request)
    if closest_request:
      closest_request_lines = closest_request.formatted_request.split('\n')
      return '\n'.join(difflib.ndiff(closest_request_lines, request_lines))
    return None

  def get_server_cert(self, host):
    """Gets certificate from the server and stores it in archive"""
    request = ArchivedHttpRequest('SERVER_CERT', host, '', None, {})
    if request not in self:
      self[request] = create_response(200, body=certutils.get_host_cert(host))
    return self[request].response_data[0]

  def get_certificate(self, host):
    request = ArchivedHttpRequest('DUMMY_CERT', host, '', None, {})
    if request not in self:
      self[request] = create_response(200, body=self._generate_cert(host))
    return self[request].response_data[0]

  @classmethod
  def AssertWritable(cls, filename):
    """Raises an IOError if filename is not writable."""
    persist_dir = os.path.dirname(os.path.abspath(filename))
    if not os.path.exists(persist_dir):
      raise IOError('Directory does not exist: %s' % persist_dir)
    if os.path.exists(filename):
      if not os.access(filename, os.W_OK):
        raise IOError('Need write permission on file: %s' % filename)
    elif not os.access(persist_dir, os.W_OK):
      raise IOError('Need write permission on directory: %s' % persist_dir)

  @classmethod
  def Load(cls, filename):
    """Load an instance from filename."""
    return cPickle.load(open(filename, 'rb'))

  def Persist(self, filename):
    """Persist all state to filename."""
    try:
      original_checkinterval = sys.getcheckinterval()
      sys.setcheckinterval(2**31-1)  # Lock out other threads so nothing can
                                     # modify |self| during pickling.
      pickled_self = cPickle.dumps(self, cPickle.HIGHEST_PROTOCOL)
    finally:
      sys.setcheckinterval(original_checkinterval)
    with open(filename, 'wb') as f:
      f.write(pickled_self)


class ArchivedHttpRequest(object):
  """Record all the state that goes into a request.

  ArchivedHttpRequest instances are considered immutable so they can
  serve as keys for HttpArchive instances.
  (The immutability is not enforced.)

  Upon creation, the headers are "trimmed" (i.e. edited or dropped)
  and saved to self.trimmed_headers to allow requests to match in a wider
  variety of playback situations (e.g. using different user agents).

  For unpickling, 'trimmed_headers' is recreated from 'headers'. That
  allows for changes to the trim function and can help with debugging.
  """
  CONDITIONAL_HEADERS = [
      'if-none-match', 'if-match',
      'if-modified-since', 'if-unmodified-since']

  def __init__(self, command, host, full_path, request_body, headers,
               is_ssl=False):
    """Initialize an ArchivedHttpRequest.

    Args:
      command: a string (e.g. 'GET' or 'POST').
      host: a host name (e.g. 'www.google.com').
      full_path: a request path.  Includes everything after the host & port in
          the URL (e.g. '/search?q=dogs').
      request_body: a request body string for a POST or None.
      headers: {key: value, ...} where key and value are strings.
      is_ssl: a boolean which is True iff request is make via SSL.
    """
    self.command = command
    self.host = host
    self.full_path = full_path
    parsed_url = urlparse.urlparse(full_path) if full_path else None
    self.path = parsed_url.path if parsed_url else None
    self.request_body = request_body
    self.headers = headers
    self.is_ssl = is_ssl
    self.trimmed_headers = self._TrimHeaders(headers)
    self.formatted_request = self._GetFormattedRequest()
    self.cmp_seq = self._GetCmpSeq(parsed_url.query if parsed_url else None)

  def __str__(self):
    scheme = 'https' if self.is_ssl else 'http'
    return '%s %s://%s%s %s' % (
        self.command, scheme, self.host, self.full_path, self.trimmed_headers)

  def __repr__(self):
    return repr((self.command, self.host, self.full_path, self.request_body,
                 self.trimmed_headers, self.is_ssl))

  def __hash__(self):
    """Return a integer hash to use for hashed collections including dict."""
    return hash(repr(self))

  def __eq__(self, other):
    """Define the __eq__ method to match the hash behavior."""
    return repr(self) == repr(other)

  def __setstate__(self, state):
    """Influence how to unpickle.

    "headers" are the original request headers.
    "trimmed_headers" are the trimmed headers used for matching requests
    during replay.

    Args:
      state: a dictionary for __dict__
    """
    if 'full_headers' in state:
      # Fix older version of archive.
      state['headers'] = state['full_headers']
      del state['full_headers']
    if 'headers' not in state:
      raise HttpArchiveException(
          'Archived HTTP request is missing "headers". The HTTP archive is'
          ' likely from a previous version and must be re-recorded.')
    if 'path' in state:
      # before, 'path' and 'path_without_query' were used and 'path' was
      # pickled.  Now, 'path' has been renamed to 'full_path' and
      # 'path_without_query' has been renamed to 'path'.  'full_path' is
      # pickled, but 'path' is not.  If we see 'path' here it means we are
      # dealing with an older archive.
      state['full_path'] = state['path']
      del state['path']
    state['trimmed_headers'] = self._TrimHeaders(dict(state['headers']))
    if 'is_ssl' not in state:
      state['is_ssl'] = False
    self.__dict__.update(state)
    parsed_url = urlparse.urlparse(self.full_path)
    self.path = parsed_url.path
    self.formatted_request = self._GetFormattedRequest()
    self.cmp_seq = self._GetCmpSeq(parsed_url.query)

  def __getstate__(self):
    """Influence how to pickle.

    Returns:
      a dict to use for pickling
    """
    state = self.__dict__.copy()
    del state['trimmed_headers']
    del state['path']
    del state['formatted_request']
    del state['cmp_seq']
    return state

  def _GetFormattedRequest(self):
    """Format request to make diffs easier to read.

    Returns:
      A string consisting of the request. Example:
      'GET www.example.com/path\nHeader-Key: header value\n'
    """
    parts = ['%s %s%s\n' % (self.command, self.host, self.full_path)]
    if self.request_body:
      parts.append('%s\n' % self.request_body)
    for k, v in self.trimmed_headers:
      k = '-'.join(x.capitalize() for x in k.split('-'))
      parts.append('%s: %s\n' % (k, v))
    return ''.join(parts)

  def _GetCmpSeq(self, query=None):
    """Compute a sequence out of query and header for difflib to compare.
    For example:
      [('q1', 'a1'), ('q2', 'a2'), ('k1', 'v1'), ('k2', 'v2')]
    will be returned for a request with URL:
      http://example.com/index.html?q1=a2&q2=a2
    and header:
      k1: v1
      k2: v2

    Args:
      query: the query string in the URL.

    Returns:
      A sequence for difflib to compare.
    """
    if not query:
      return self.trimmed_headers
    return sorted(urlparse.parse_qsl(query)) + self.trimmed_headers

  def matches(self, command=None, host=None, full_path=None, is_ssl=None,
              use_query=True):
    """Returns true iff the request matches all parameters.

    Args:
      command: a string (e.g. 'GET' or 'POST').
      host: a host name (e.g. 'www.google.com').
      full_path: a request path with query string (e.g. '/search?q=dogs')
      is_ssl: whether the request is secure.
      use_query:
        If use_query is True, request matching uses both the hierarchical path
        and query string component.
        If use_query is False, request matching only uses the hierarchical path

        e.g. req1 = GET www.test.com/index?aaaa
             req2 = GET www.test.com/index?bbbb

        If use_query is True, req1.matches(req2) evaluates to False
        If use_query is False, req1.matches(req2) evaluates to True

    Returns:
      True iff the request matches all parameters
    """
    if command is not None and command != self.command:
      return False
    if is_ssl is not None and is_ssl != self.is_ssl:
      return False
    if host is not None and host != self.host:
      return False
    if full_path is None:
      return True
    if use_query:
      return full_path == self.full_path
    else:
      return self.path == urlparse.urlparse(full_path).path

  @classmethod
  def _TrimHeaders(cls, headers):
    """Removes headers that are known to cause problems during replay.

    These headers are removed for the following reasons:
    - accept: Causes problems with www.bing.com. During record, CSS is fetched
              with *. During replay, it's text/css.
    - accept-charset, accept-language, referer: vary between clients.
    - cache-control:  sometimes sent from Chrome with 'max-age=0' as value.
    - connection, method, scheme, url, version: Cause problems with spdy.
    - cookie: Extremely sensitive to request/response order.
    - keep-alive: Doesn't affect the content of the request, only some
      transient state of the transport layer.
    - user-agent: Changes with every Chrome version.
    - proxy-connection: Sent for proxy requests.
    - x-chrome-variations, x-client-data: Unique to each Chrome binary. Used by
      Google to collect statistics about Chrome's enabled features.

    Another variant to consider is dropping only the value from the header.
    However, this is particularly bad for the cookie header, because the
    presence of the cookie depends on the responses we've seen when the request
    is made.

    Args:
      headers: {header_key: header_value, ...}

    Returns:
      [(header_key, header_value), ...]  # (with undesirable headers removed)
    """
    # TODO(tonyg): Strip sdch from the request headers because we can't
    # guarantee that the dictionary will be recorded, so replay may not work.
    if 'accept-encoding' in headers:
      accept_encoding = headers['accept-encoding']
      accept_encoding = accept_encoding.replace('sdch', '')
      # Strip lzma so Opera's requests matches archives recorded using Chrome.
      accept_encoding = accept_encoding.replace('lzma', '')
      stripped_encodings = [e.strip() for e in accept_encoding.split(',')]
      accept_encoding = ','.join(filter(bool, stripped_encodings))
      headers['accept-encoding'] = accept_encoding
    undesirable_keys = [
        'accept', 'accept-charset', 'accept-language', 'cache-control',
        'connection', 'cookie', 'keep-alive', 'method',
        'referer', 'scheme', 'url', 'version', 'user-agent', 'proxy-connection',
        'x-chrome-variations', 'x-client-data']
    return sorted([(k, v) for k, v in headers.items()
                   if k.lower() not in undesirable_keys])

  def is_conditional(self):
    """Return list of headers that match conditional headers."""
    for header in self.CONDITIONAL_HEADERS:
      if header in self.headers:
        return True
    return False

  def create_request_without_conditions(self):
    stripped_headers = dict((k, v) for k, v in self.headers.iteritems()
                            if k.lower() not in self.CONDITIONAL_HEADERS)
    return ArchivedHttpRequest(
        self.command, self.host, self.full_path, self.request_body,
        stripped_headers, self.is_ssl)

class ArchivedHttpResponse(object):
  """All the data needed to recreate all HTTP response.

  Upon creation, the headers are "trimmed" (i.e. edited or dropped).
  The original headers are saved to self.original_headers, while the
  trimmed ones are used to allow responses to match in a wider variety
  of playback situations.

  For pickling, 'original_headers' are stored in the archive.  For unpickling
  the headers are trimmed again. That allows for changes to the trim
  function and can help with debugging.
  """

  # CHUNK_EDIT_SEPARATOR is used to edit and view text content.
  # It is not sent in responses. It is added by get_data_as_text()
  # and removed by set_data().
  CHUNK_EDIT_SEPARATOR = '[WEB_PAGE_REPLAY_CHUNK_BOUNDARY]'

  # DELAY_EDIT_SEPARATOR is used to edit and view server delays.
  DELAY_EDIT_SEPARATOR = ('\n[WEB_PAGE_REPLAY_EDIT_ARCHIVE --- '
                          'Delays are above. Response content is below.]\n')

  def __init__(self, version, status, reason, headers, response_data,
               delays=None):
    """Initialize an ArchivedHttpResponse.

    Args:
      version: HTTP protocol version used by server.
          10 for HTTP/1.0, 11 for HTTP/1.1 (same as httplib).
      status: Status code returned by server (e.g. 200).
      reason: Reason phrase returned by server (e.g. "OK").
      headers: list of (header, value) tuples.
      response_data: list of content chunks.
          Concatenating the chunks gives the complete contents
          (i.e. the chunks do not have any lengths or delimiters).
          Do not include the final, zero-length chunk that marks the end.
      delays: dict of (ms) delays for 'connect', 'headers' and 'data'.
          e.g. {'connect': 50, 'headers': 150, 'data': [0, 10, 10]}
          connect - The time to connect to the server.
            Each resource has a value because Replay's record mode captures it.
            This includes the time for the SYN and SYN/ACK (1 rtt).
          headers - The time elapsed between the TCP connect and the headers.
            This typically includes all the server-time to generate a response.
          data - If the response is chunked, these are the times for each chunk.
    """
    self.version = version
    self.status = status
    self.reason = reason
    self.original_headers = headers
    self.headers = self._TrimHeaders(headers)
    self.response_data = response_data
    self.delays = delays
    self.fix_delays()

  def fix_delays(self):
    """Initialize delays, or check the number of data delays."""
    expected_num_delays = len(self.response_data)
    if not self.delays:
      self.delays = {
          'connect': 0,
          'headers': 0,
          'data': [0] * expected_num_delays
          }
    else:
      num_delays = len(self.delays['data'])
      if num_delays != expected_num_delays:
        raise HttpArchiveException(
            'Server delay length mismatch: %d (expected %d): %s',
            num_delays, expected_num_delays, self.delays['data'])

  @classmethod
  def _TrimHeaders(cls, headers):
    """Removes headers that are known to cause problems during replay.

    These headers are removed for the following reasons:
    - content-security-policy: Causes problems with script injection.
    """
    undesirable_keys = ['content-security-policy']
    return [(k, v) for k, v in headers if k.lower() not in undesirable_keys]

  def __repr__(self):
    return repr((self.version, self.status, self.reason, sorted(self.headers),
                 self.response_data))

  def __hash__(self):
    """Return a integer hash to use for hashed collections including dict."""
    return hash(repr(self))

  def __eq__(self, other):
    """Define the __eq__ method to match the hash behavior."""
    return repr(self) == repr(other)

  def __setstate__(self, state):
    """Influence how to unpickle.

    "original_headers" are the original request headers.
    "headers" are the trimmed headers used for replaying responses.

    Args:
      state: a dictionary for __dict__
    """
    if 'server_delays' in state:
      state['delays'] = {
          'connect': 0,
          'headers': 0,
          'data': state['server_delays']
          }
      del state['server_delays']
    elif 'delays' not in state:
      state['delays'] = None
    state['original_headers'] = state['headers']
    state['headers'] = self._TrimHeaders(state['original_headers'])
    self.__dict__.update(state)
    self.fix_delays()

  def __getstate__(self):
    """Influence how to pickle.

    Returns:
      a dict to use for pickling
    """
    state = self.__dict__.copy()
    state['headers'] = state['original_headers']
    del state['original_headers']
    return state

  def get_header(self, key, default=None):
    for k, v in self.headers:
      if key.lower() == k.lower():
        return v
    return default

  def set_header(self, key, value):
    for i, (k, v) in enumerate(self.headers):
      if key == k:
        self.headers[i] = (key, value)
        return
    self.headers.append((key, value))

  def remove_header(self, key):
    for i, (k, v) in enumerate(self.headers):
      if key.lower() == k.lower():
        self.headers.pop(i)
        return

  @staticmethod
  def _get_epoch_seconds(date_str):
    """Return the epoch seconds of a date header.

    Args:
      date_str: a date string (e.g. "Thu, 01 Dec 1994 16:00:00 GMT")
    Returns:
      epoch seconds as a float
    """
    date_tuple = email.utils.parsedate(date_str)
    if date_tuple:
      return calendar.timegm(date_tuple)
    return None

  def update_date(self, date_str, now=None):
    """Return an updated date based on its delta from the "Date" header.

    For example, if |date_str| is one week later than the "Date" header,
    then the returned date string is one week later than the current date.

    Args:
      date_str: a date string (e.g. "Thu, 01 Dec 1994 16:00:00 GMT")
    Returns:
      a date string
    """
    date_seconds = self._get_epoch_seconds(self.get_header('date'))
    header_seconds = self._get_epoch_seconds(date_str)
    if date_seconds and header_seconds:
      updated_seconds = header_seconds + (now or time.time()) - date_seconds
      return email.utils.formatdate(updated_seconds, usegmt=True)
    return date_str

  def is_gzip(self):
    return self.get_header('content-encoding') == 'gzip'

  def is_compressed(self):
    return self.get_header('content-encoding') in ('gzip', 'deflate')

  def is_chunked(self):
    return self.get_header('transfer-encoding') == 'chunked'

  def get_data_as_text(self):
    """Return content as a single string.

    Uncompresses and concatenates chunks with CHUNK_EDIT_SEPARATOR.
    """
    content_type = self.get_header('content-type')
    if (not content_type or
        not (content_type.startswith('text/') or
             content_type == 'application/x-javascript' or
             content_type.startswith('application/json'))):
      return None
    if self.is_compressed():
      uncompressed_chunks = httpzlib.uncompress_chunks(
          self.response_data, self.is_gzip())
    else:
      uncompressed_chunks = self.response_data
    return self.CHUNK_EDIT_SEPARATOR.join(uncompressed_chunks)

  def get_delays_as_text(self):
    """Return delays as editable text."""
    return json.dumps(self.delays, indent=2)

  def get_response_as_text(self):
    """Returns response content as a single string.

    Server delays are separated on a per-chunk basis. Delays are in seconds.
    Response content begins after DELAY_EDIT_SEPARATOR
    """
    data = self.get_data_as_text()
    if data is None:
      logging.warning('Data can not be represented as text.')
      data = ''
    delays = self.get_delays_as_text()
    return self.DELAY_EDIT_SEPARATOR.join((delays, data))

  def set_data(self, text):
    """Inverse of get_data_as_text().

    Split on CHUNK_EDIT_SEPARATOR and compress if needed.
    """
    text_chunks = text.split(self.CHUNK_EDIT_SEPARATOR)
    if self.is_compressed():
      self.response_data = httpzlib.compress_chunks(text_chunks, self.is_gzip())
    else:
      self.response_data = text_chunks
    if not self.is_chunked():
      content_length = sum(len(c) for c in self.response_data)
      self.set_header('content-length', str(content_length))

  def set_delays(self, delays_text):
    """Inverse of get_delays_as_text().

    Args:
      delays_text: JSON encoded text such as the following:
          {
            connect: 80,
            headers: 80,
            data: [6, 55, 0]
          }
        Times are in milliseconds.
        Each data delay corresponds with one response_data value.
    """
    try:
      self.delays = json.loads(delays_text)
    except (ValueError, KeyError) as e:
      logging.critical('Unable to parse delays %s: %s', delays_text, e)
    self.fix_delays()

  def set_response_from_text(self, text):
    """Inverse of get_response_as_text().

    Modifies the state of the archive according to the textual representation.
    """
    try:
      delays, data = text.split(self.DELAY_EDIT_SEPARATOR)
    except ValueError:
      logging.critical(
          'Error parsing text representation. Skipping edits.')
      return
    self.set_delays(delays)
    self.set_data(data)


def create_response(status, reason=None, headers=None, body=None):
  """Convenience method for creating simple ArchivedHttpResponse objects."""
  if reason is None:
    reason = httplib.responses.get(status, 'Unknown')
  if headers is None:
    headers = [('content-type', 'text/plain')]
  if body is None:
    body = "%s %s" % (status, reason)
  return ArchivedHttpResponse(11, status, reason, headers, [body])


def main():
  class PlainHelpFormatter(optparse.IndentedHelpFormatter):
    def format_description(self, description):
      if description:
        return description + '\n'
      else:
        return ''

  option_parser = optparse.OptionParser(
      usage='%prog [ls|cat|edit|stats|merge] [options] replay_file(s)',
      formatter=PlainHelpFormatter(),
      description=__doc__,
      epilog='http://code.google.com/p/web-page-replay/')

  option_parser.add_option('-c', '--command', default=None,
      action='store',
      type='string',
      help='Only show URLs matching this command.')
  option_parser.add_option('-o', '--host', default=None,
      action='store',
      type='string',
      help='Only show URLs matching this host.')
  option_parser.add_option('-p', '--full_path', default=None,
      action='store',
      type='string',
      help='Only show URLs matching this full path.')
  option_parser.add_option('-f', '--merged_file', default=None,
        action='store',
        type='string',
        help='The output file to use when using the merge command.')

  options, args = option_parser.parse_args()

  # Merge command expects an umlimited number of archives.
  if len(args) < 2:
    print 'args: %s' % args
    option_parser.error('Must specify a command and replay_file')

  command = args[0]
  replay_file = args[1]

  if not os.path.exists(replay_file):
    option_parser.error('Replay file "%s" does not exist' % replay_file)

  http_archive = HttpArchive.Load(replay_file)
  if command == 'ls':
    print http_archive.ls(options.command, options.host, options.full_path)
  elif command == 'cat':
    print http_archive.cat(options.command, options.host, options.full_path)
  elif command == 'stats':
    print http_archive.stats(options.command, options.host, options.full_path)
  elif command == 'merge':
    if not options.merged_file:
      print 'Error: Must specify a merged file name (use --merged_file)'
      return
    http_archive.merge(options.merged_file, args[2:])
  elif command == 'edit':
    http_archive.edit(options.command, options.host, options.full_path)
    http_archive.Persist(replay_file)
  else:
    option_parser.error('Unknown command "%s"' % command)
  return 0


if __name__ == '__main__':
  sys.exit(main())