Migration to Stage1 format

.travis.yml

@@ -21,7 +21,7 @@ install:
       - conda update -q conda
         # Useful for debugging any issues with conda
       - conda info -a
-      - conda create -n test python=3.7
+      - conda create -n test python=3.7 cython
       - conda activate test
       - python setup.py develop
 script:

README.md

@@ -45,10 +45,10 @@ python setup_light.py install
 
 To install it as a conda package, first install Anaconda by following the instructions here: https://www.anaconda.com/distribution/.
 
-Then, create and enter a new Python 3.7 environment with:
+Then, create and enter a new Python 3.7 (with cython) environment with:
 
 ```bash
-conda create -n [ENVIRONMENT_NAME] python=3.7
+conda create -n [ENVIRONMENT_NAME] python=3.7 cython
 conda activate [ENVIRONMENT_NAME]
 ```
 
@@ -83,7 +83,7 @@ The main dependencies are:
 
 * PyTables >= 3.4.4
 * NumPy >= 1.15.0
-* ctapipe == 0.9.1
+* ctapipe == 0.10.3
 
 Also see setup.py.
 
@@ -96,14 +96,14 @@ Also see setup.py.
 To process data files into a desired format:
 
 ```bash
-scripts/write_data.py [runlist] [--config_file CONFIG_FILE_PATH] [--output_dir OUTPUT_DIR] [--debug]
+dl1dh-write_data [runlist] [--config_file,-c CONFIG_FILE_PATH] [--output_dir,-o OUTPUT_DIR] [--debug]
 ```
 on the command line.
 
 ex:
 
 ```bash
-scripts/write_data.py runlist.yml --config_file example_config.yml --debug
+dl1dh-write_data runlist.yml -c example_config.yml --debug
 ```
 
 * runlist - A YAML file containing groups of input files to load data from and output files to write to. See example runlist for format.
@@ -148,17 +148,18 @@ data_writer.process_data(run_list)
 ```
 #### Generating a run list
 
-If processing data from simtel.gz files, as long as their filenames have the format ``[particle_type]_[ze]deg_[az]deg_run[run_number]___[production info].simtel.gz`` or ``[particle_type]_[ze]deg_[az]deg_run[run_number]___[production info]_cone[cone_num].simtel.gz`` the scripts/generate_runlist.py can be used to automatically generate a runlist in the correct format.
+If processing data from simtel.gz files, as long as their filenames have the format ``[particle_type]_[ze]deg_[az]deg_run[run_number]___[production info].simtel.gz`` or ``[particle_type]_[ze]deg_[az]deg_run[run_number]___[production info]_cone[cone_num].simtel.gz`` the dl1dh-generate_runlist can be used to automatically generate a runlist in the correct format. The script can also generate a run list with the MAGIC-MARS superstar files.
 
 It can be called as:
 
 ```bash
-scripts/generate_runlist.py [file_dir] [--num_inputs_per_run NUM_INPUTS_PER_RUN] [--output_file OUTPUT_FILE]
+dl1dh-generate_runlist [file_dir] [--num_inputs_per_run,-n NUM_INPUTS_PER_RUN] [--output_file_name,-f OUTPUT_FILE_NAME] [--output_dir,-o OUTPUT_DIR]
 ```
 
 * file_dir - Path to a directory containing simtel.gz files with the filename format specified above.
 * num_inputs_per_run - Number of input files with the same particle type, ze, az, and production info to group together into each run (defaults to 10).
-* output_file - Path/filename of output runlist file. Defaults to ./runlist.yml
+* output_file - Path/filename of output runlist file without a postfix. Defaults to ./runlist
+* output_dir - Path where to save generated files. By default, the input directory is used.
 
 It will automatically sort the simtel files in the file_dir directory into groups with matching particle_type, zenith, azimuth, and production parameters. Within each of these groups, it will group together input files in sequential order into runs of size NUM_INPUTS_PER_RUN. The output filename for each run will be automatically generated as ``[particle_type]_[ze]deg_[az]deg_runs[run_number_range]___[production info].h5``. The output YAML file will be written to output_file.
 

config/example_config.yml

@@ -14,7 +14,7 @@ Data Writer:
     # Boolean flag indicating whether to save non-triggered Monte Carlo events in a separate MC_Events table.
     save_mc_events: False
     # Telescope ids dumped to file. Specify the telescopes to write to the the output file.
-    selected_telescope_ids: [1]
+    selected_telescope_ids: [1,2]
     #selected_telescope_ids: [1,2,3,4]
     # Settinigs for the GainSelector
     gain_selector_settings:

dl1_data_handler/containers.py

@@ -10,6 +10,8 @@ class MAGICMCHeaderContainer(Container):
     prod_site = Field(nan, "Production site")
     date_run_mmcs = Field(nan, "Date Run MMCs")
     date_run_cam = Field(nan, "Date Run Camera")
+    energy_range_max = Field(nan * u.GeV, "Energy range Maximum", unit = u.GeV)
+    energy_range_min = Field(nan * u.GeV, "Energy range Minimum", unit = u.GeV)
     shower_theta_max = Field(nan * u.deg, "Shower Theta Maximum", unit = u.deg)
     shower_theta_min = Field(nan * u.deg, "Shower Theta Minimum", unit = u.deg)
     shower_phi_max = Field(nan * u.deg, "Shower Phi Maximum", unit = u.deg)
@@ -20,7 +22,7 @@ class MAGICMCHeaderContainer(Container):
     height_lev = Field([], "Height Level")
     slope_spec = Field(nan, "Slope Spec")
     rand_pointing_cone_semi_angle = Field(nan * u.deg, "Random Pointing Cone Semi Angle", unit = u.deg)
-    impact_max = Field(nan, "Imapact Maximum")
+    impact_max = Field(nan, "Impact Maximum")
     star_field_rotate = Field(nan, "Star Field Rotate")
     star_field_ra_h = Field(nan, "Star Field RA H")
     star_field_ra_m = Field(nan, "Star Field RA M")
@@ -41,3 +43,44 @@ class MAGICMCHeaderContainer(Container):
     num_phe_from_dnsb = Field(nan, "Number Phe from DNSB")
     elec_noise = Field(nan, "Elec Noise")
     optic_links_noise = Field(nan, "Optic Links Noise")
+
+class MAGICHeaderContainer(Container):
+    camera_version = Field(nan, "Camera Version")
+    fadc_type = Field(nan, "FADC type")
+    fadc_resolution = Field(nan, "FADC resolution")
+    format_version = Field(nan, "Format version")
+    magic_number = Field(nan, "MAGIC number")
+    num_bytes_per_sample = Field(nan, "Number of bytes per sample")
+    num_crates = Field(nan, "Number of crates")
+    num_pix_in_crate = Field(nan, "Number of pixels per crate")
+    num_samples_hi_gain = Field(nan, "Number of High gain samples")
+    num_samples_lo_gain = Field(nan, "Number of Low gain samples")
+    num_samples_removed_head = Field(nan, "Number of samples removed from the head")
+    num_samples_removed_tail = Field(nan, "Number of samples removed from the tail")
+    run_type = Field(nan, "Run type")
+    online_domino_calib = Field(nan, "Online domino calibration")
+    sample_frequency = Field(nan, "Sample frequency")
+    soft_version = Field(nan, "Soft version")
+    source_epoch_date = Field(nan, "Source epoch date")
+    num_events = Field(nan, "Number of events")
+    num_events_read = Field(nan, "Number of events read")
+    channel_header_size = Field(nan, "Channel header size")
+    event_header_size = Field(nan, "Event header size")
+    run_header_size = Field(nan, "Run header size")
+    run_number = Field(nan, "Run number")
+    subrun_index = Field(nan, "Subrun index")
+    source_dec = Field(nan, "Source Dec")
+    source_ra = Field(nan, "Source RA")
+    telescope_dec = Field(nan, "Telescope Dec")
+    telescope_ra = Field(nan, "Telescope RA")
+    observation_mode = Field(nan, "Observation mode")
+    project_name = Field(nan, "Project name")
+    source_epoch_char = Field(nan, "Source epoch char")
+    source_name = Field(nan, "Source name")
+    calib_coeff_filename = Field(nan, "Calibration coefficient filename")
+    run_start_mjd = Field(nan, "Start date MJD")
+    run_start_ms = Field(nan, "Start time in ms")
+    run_start_ns = Field(nan, "Start time in ns")
+    run_stop_mjd = Field(nan, "Stop date MJD")
+    run_stop_ms = Field(nan, "Stop time in ms")
+    run_stop_ns = Field(nan, "Stop time in ns")