From 2b3ca7f868789677ceb4e6f73045e494923a10de Mon Sep 17 00:00:00 2001
From: Alex Tait <alexander.tait@nist.gov>
Date: Fri, 16 Nov 2018 08:23:24 -0700
Subject: [PATCH 1/6] described problem. allowed changing target host

---
 README.md                | 46 ++++++++++++++++++++++++++++++++++++++++
 examples/.gitignore      |  3 ++-
 lyipc/client/__init__.py | 16 ++++++++++----
 lyipc/interpreter.py     |  5 ++++-
 4 files changed, 64 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 1bfed2c..7a09c78 100644
--- a/README.md
+++ b/README.md
@@ -102,5 +102,51 @@ The former is faster because a new klayout instance is not created, but of cours
 
 Usage examples for klayout and non-klayout layout clients are included in this repo in the "examples" folder.
 
+
+## Long shot wishlist item: remote jobs
+Tests run a lot. Integration tests of entire wafers can take a really long time. Yet they can be parallelized – on the good side of Amdahl's law. This also has relevance to dataprep
+
+Here is the process
+
+1. [laptop] initiates klayout IPC configured for incoming connections (firewalled)
+1. [laptop (or CI)] command for test
+1. [laptop] sends IPC port info to remote HPC
+1. [laptop] rsyncs ref_layouts and the new code, changes only, to the remote computer
+    - you might also have to synchronize lygadgets, or just do it manually
+    - can we override rsync's archive detection with geometry instead of binary
+    - no it is based on file modification time, so we are good
+1. [laptop] sends command to remote computer to initiate test
+1. [remote HPC] runs tests on all cores using pytest-xdist and klayout tiling
+1. [remote HPC] sends progress reports
+    - pipe pytest output by redirecting stdout?
+1. [remote HPC] if there are errors, sends the layout pairs to the IPC server on the laptop
+1. [remote HPC] sends some kind of completion signal
+1. [laptop] rsyncs run_layouts for further examination
+    - only the failures?
+
+Following are the steps to enabling this
+
+#### Network IPC (Done)
+Run a server on one computer. Configure something in lyipc in the second computer. Send lyipc commands. At first, do load with the gds already on the first computer.
+
+#### stdout piping
+Test script will look something like
+```python
+with redirect_stdout():
+    print('heyo')
+```
+This script should be initiated by the laptop
+
+#### file transfer and IPC
+Set some configuration of lytest, which sets some configuration of lyipc. Run `lytest diff file1.gds file2.gds`. These files are shipped to remote. XOR is run there. Error is detected and sent back to the klayout GUI of the first computer. This will involve actual file transfer.
+
+#### script building
+lytest not only sends a ref_layout but also a script. This scripted layout is built remotely. The XOR is done remotely.
+
+#### tiling
+To take full advantage, we eventually want to distribute tiles over cores. At first, we will get good results from xdist alone... when it comes to testing, but not dataprep.
+
+
+
 #### Author: Alex Tait, June 2018
 #### National Institute of Standards and Technology, Boulder, CO, USA
diff --git a/examples/.gitignore b/examples/.gitignore
index 1b52038..0afca9f 100644
--- a/examples/.gitignore
+++ b/examples/.gitignore
@@ -1 +1,2 @@
-*.gds
\ No newline at end of file
+*.gds
+remote_jobs
\ No newline at end of file
diff --git a/lyipc/client/__init__.py b/lyipc/client/__init__.py
index 09ffbdc..5d95359 100644
--- a/lyipc/client/__init__.py
+++ b/lyipc/client/__init__.py
@@ -8,6 +8,7 @@
 
 import lyipc
 from lygadgets import isGSI
+import socket
 
 
 # Determine which socket class to use
@@ -19,11 +20,18 @@
     elif sys.version_info[0] == 3:
         import PyQt5.QtNetwork
         from PyQt5.QtNetwork import QTcpSocket
-        localhost = PyQt5.QtNetwork.QHostAddress.LocalHost
+        # localhost = '127.0.0.1'
 else:
     from lygadgets import pya
     from pya import QTcpSocket
-    localhost = 'localhost'
+    # localhost = 'localhost'
+
+target_host = None
+def set_target_hostname(hostname_or_ip):
+    global target_host
+    target_host = socket.gethostbyname(hostname_or_ip)
+
+set_target_hostname('localhost')
 
 
 def send(message='ping 1234', port=lyipc.PORT):
@@ -35,7 +43,7 @@ def send(message='ping 1234', port=lyipc.PORT):
     psock = QTcpSocket()
     if not isGSI():
         payload = payload.encode()
-    psock.connectToHost(localhost, port)
+    psock.connectToHost(target_host, port)
     if psock.waitForConnected():
         psock.write(payload)
         if psock.waitForReadyRead(3000):
@@ -46,7 +54,7 @@ def send(message='ping 1234', port=lyipc.PORT):
         else:
             raise Exception('Not acknowledged')
     else:
-        print('Connection Fail! (tried {}:{})'.format(localhost, port))
+        print('Connection Fail! (tried {}:{})'.format(target_host, port))
     # psock.close()
 
 
diff --git a/lyipc/interpreter.py b/lyipc/interpreter.py
index 2192580..e4bbe19 100644
--- a/lyipc/interpreter.py
+++ b/lyipc/interpreter.py
@@ -6,7 +6,7 @@
         - execute a macro
 '''
 from __future__ import print_function
-from lygadgets import message, isGSI
+from lygadgets import message, message_loud, isGSI
 import lyipc.server
 import os
 import traceback
@@ -55,6 +55,9 @@ def parse_message(msg):
             main = pya.Application.instance().main_window()
             main.cm_reload()
 
+        elif tokens[0] == 'ping':
+            message_loud('I heard something')
+
         elif tokens[0] == 'load':
             filename = os.path.realpath(tokens[1])
             message(filename)

From 7fb6f0457940132dda47b753caaa0c07afa8dce2 Mon Sep 17 00:00:00 2001
From: Alex Tait <alexander.tait@nist.gov>
Date: Sat, 17 Nov 2018 22:31:19 -0700
Subject: [PATCH 2/6] working on sending files over rsync

---
 README.md                  |  16 ++++--
 lyipc/client/__init__.py   |  14 ++---
 lyipc/client/remotehost.py | 102 +++++++++++++++++++++++++++++++++++++
 3 files changed, 119 insertions(+), 13 deletions(-)
 create mode 100644 lyipc/client/remotehost.py

diff --git a/README.md b/README.md
index 7a09c78..d428935 100644
--- a/README.md
+++ b/README.md
@@ -127,7 +127,7 @@ Here is the process
 Following are the steps to enabling this
 
 #### Network IPC (Done)
-Run a server on one computer. Configure something in lyipc in the second computer. Send lyipc commands. At first, do load with the gds already on the first computer.
+Run a server on one computer. Configure something in lyipc in the second computer. Send lyipc commands. At first, do load with the gds already on the first computer. Next, combine with rsync and gds on local computer with client.load
 
 #### stdout piping
 Test script will look something like
@@ -135,9 +135,19 @@ Test script will look something like
 with redirect_stdout():
     print('heyo')
 ```
-This script should be initiated by the laptop
+This script should be initiated by the laptop but run on the HPC.
 
-#### file transfer and IPC
+I got this working, but its not live.
+
+#### remote build
+1. [laptop user] lyipc-job script.py
+1. [laptop] rsync script.py
+1. [HPC] python script.py
+1. [HPC] rsync output.gds
+
+Should this use container functions?
+
+#### file transfer and IPC and lytest
 Set some configuration of lytest, which sets some configuration of lyipc. Run `lytest diff file1.gds file2.gds`. These files are shipped to remote. XOR is run there. Error is detected and sent back to the klayout GUI of the first computer. This will involve actual file transfer.
 
 #### script building
diff --git a/lyipc/client/__init__.py b/lyipc/client/__init__.py
index 5d95359..37e59e9 100644
--- a/lyipc/client/__init__.py
+++ b/lyipc/client/__init__.py
@@ -9,7 +9,8 @@
 import lyipc
 from lygadgets import isGSI
 import socket
-
+import os
+from lyipc.client.remotehost import set_target_hostname, get_target_hostname
 
 # Determine which socket class to use
 if not isGSI():
@@ -26,13 +27,6 @@
     from pya import QTcpSocket
     # localhost = 'localhost'
 
-target_host = None
-def set_target_hostname(hostname_or_ip):
-    global target_host
-    target_host = socket.gethostbyname(hostname_or_ip)
-
-set_target_hostname('localhost')
-
 
 def send(message='ping 1234', port=lyipc.PORT):
     ''' Sends a raw message
@@ -43,7 +37,7 @@ def send(message='ping 1234', port=lyipc.PORT):
     psock = QTcpSocket()
     if not isGSI():
         payload = payload.encode()
-    psock.connectToHost(target_host, port)
+    psock.connectToHost(get_target_hostname(), port)
     if psock.waitForConnected():
         psock.write(payload)
         if psock.waitForReadyRead(3000):
@@ -54,7 +48,7 @@ def send(message='ping 1234', port=lyipc.PORT):
         else:
             raise Exception('Not acknowledged')
     else:
-        print('Connection Fail! (tried {}:{})'.format(target_host, port))
+        print('Connection Fail! (tried {}:{})'.format(get_target_hostname(), port))
     # psock.close()
 
 
diff --git a/lyipc/client/remotehost.py b/lyipc/client/remotehost.py
new file mode 100644
index 0000000..6b8b54e
--- /dev/null
+++ b/lyipc/client/remotehost.py
@@ -0,0 +1,102 @@
+import os
+import shutil
+import socket
+import subprocess
+
+target_host = None
+def set_target_hostname(hostalias, persist=False):
+    ''' if it is a remote, you must have already set up an RSA key and alias in your ~/.ssh/config file.
+        On that computer, this computer's RSA key needs to be in ~/.ssh/authorized_keys.
+        Instructions: https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys--2
+    '''
+    global target_host
+    if hostalias == 'localhost':
+        hostalias = socket.gethostbyname(hostalias)
+    target_host = hostalias
+    if persist:
+        os.environ['LYIPCTARGET'] = target_host
+
+
+def get_target_hostname():
+    try:
+        return os.environ['LYIPCTARGET']
+    except KeyError:
+        return target_host
+
+
+set_target_hostname('localhost')
+
+
+def is_host_remote():
+    return get_target_hostname() != socket.gethostbyname('localhost')
+
+
+def call_report(command, verbose=True):
+    if verbose:
+        print = lambda *args: None
+    print('\n[[' + ' '.join(command) + ']]\n')
+    try:
+        ret = subprocess.check_output(command).decode()
+    except subprocess.CalledProcessError as err:
+        print(err.output.decode())
+        raise
+    else:
+        print(ret)
+        return ret
+
+
+def call_ssh(command):
+    # command[0] = '"' + command[0]
+    # command[-1] = command[-1] + '"'
+    ssh_command = ['ssh', '-t', get_target_hostname()] + command
+    return call_report(ssh_command)
+
+
+def host_HOME():
+    return call_ssh(['echo', '$HOME']).strip()
+
+
+def rsync(source, dest, verbose=True):
+    rsync_call = ['rsync', '-avzh']
+    rsync_call += [source, dest]
+    call_report(rsync_call, verbose=verbose)
+
+
+def ship_file(local_file):
+    ''' returns the name of the remote file
+        This currently assumes that the host has the same operating system separator as this one (e.g. "/")
+    '''
+    if not is_host_remote():
+        return local_file
+    # where are we going to put it
+    local_file = os.path.realpath(local_file)
+    # rel_filepath = os.sep.join(local_file.split(os.sep)[-3:-1])  # pick off a few directories to avoid name clashes
+    rel_filepath = ''
+    remote_path = os.path.join('~', 'tmp_lypic', rel_filepath)
+    remote_file = os.path.join(remote_path, os.path.basename(local_file))
+    # before rsyncing you have to mirror it on this computer
+    # import pdb; pdb.set_trace()
+    # local_mirror = remote_path
+    # mirror_file = os.path.join(local_mirror, os.path.basename(local_file))
+    # try:
+    #     os.mkdir(os.path.expanduser(local_mirror))
+    # except FileExistsError:
+    #     pass
+    # shutil.copy2(local_file, os.path.expanduser(local_mirror))
+    # # shutil.copy2('/Users/ant12/Documents/git-projects/layout-code/klayout-ipc/examples/box.gds',
+    # #              '/Users/ant12/tmp_lyipc/klayout-ipc/examples/')
+    # rsync(os.path.expanduser(mirror_file), get_target_hostname() + ':' + remote_path)
+    # return remote_file
+    # before rsyncing you have to mirror it on this computer
+    import pdb; pdb.set_trace()
+    local_mirror = remote_path
+    mirror_file = os.path.join(local_mirror, os.path.basename(local_file))
+    # try:
+    #     os.mkdir(os.path.expanduser(local_mirror))
+    # except FileExistsError:
+    #     pass
+    # shutil.copy2(local_file, os.path.expanduser(local_mirror))
+    # shutil.copy2('/Users/ant12/Documents/git-projects/layout-code/klayout-ipc/examples/box.gds',
+    #              '/Users/ant12/tmp_lyipc/klayout-ipc/examples/')
+    rsync(local_file, get_target_hostname() + ':' + remote_path)
+    return remote_file

From d62d19629bcb009780c8049abc48708edf62ec42 Mon Sep 17 00:00:00 2001
From: Alex Tait <alexander.tait@nist.gov>
Date: Sat, 17 Nov 2018 22:33:33 -0700
Subject: [PATCH 3/6] working without mirrors. i think the subdirectory thing
 messed it up

---
 lyipc/client/general.py    |  5 +++++
 lyipc/client/remotehost.py | 28 ++--------------------------
 2 files changed, 7 insertions(+), 26 deletions(-)

diff --git a/lyipc/client/general.py b/lyipc/client/general.py
index bccbc2d..8d77b33 100644
--- a/lyipc/client/general.py
+++ b/lyipc/client/general.py
@@ -1,6 +1,7 @@
 ''' These are functions that are independent of the layout package '''
 from __future__ import print_function
 from lyipc.client import send
+from lyipc.client.remotehost import is_host_remote, ship_file
 from functools import lru_cache
 import os
 import time
@@ -19,10 +20,14 @@ def load(filename, mode=None):
         - 1: making a new view
         - 2: adding the layout to the current view (mode 2)
     '''
+    # if it's remote, we have to ship the file over first
     filename = fast_realpath(filename)
+    if is_host_remote():
+        filename = ship_file(filename)  # filename has changed to reflect what it is on the remote system
     tokens = ['load', filename]
     if mode is not None:
         tokens.append(str(mode))
+    target
     send(' '.join(tokens))
 
 
diff --git a/lyipc/client/remotehost.py b/lyipc/client/remotehost.py
index 6b8b54e..a470bd1 100644
--- a/lyipc/client/remotehost.py
+++ b/lyipc/client/remotehost.py
@@ -72,31 +72,7 @@ def ship_file(local_file):
     local_file = os.path.realpath(local_file)
     # rel_filepath = os.sep.join(local_file.split(os.sep)[-3:-1])  # pick off a few directories to avoid name clashes
     rel_filepath = ''
-    remote_path = os.path.join('~', 'tmp_lypic', rel_filepath)
+    remote_path = os.path.join('tmp_lypic', rel_filepath)
     remote_file = os.path.join(remote_path, os.path.basename(local_file))
-    # before rsyncing you have to mirror it on this computer
-    # import pdb; pdb.set_trace()
-    # local_mirror = remote_path
-    # mirror_file = os.path.join(local_mirror, os.path.basename(local_file))
-    # try:
-    #     os.mkdir(os.path.expanduser(local_mirror))
-    # except FileExistsError:
-    #     pass
-    # shutil.copy2(local_file, os.path.expanduser(local_mirror))
-    # # shutil.copy2('/Users/ant12/Documents/git-projects/layout-code/klayout-ipc/examples/box.gds',
-    # #              '/Users/ant12/tmp_lyipc/klayout-ipc/examples/')
-    # rsync(os.path.expanduser(mirror_file), get_target_hostname() + ':' + remote_path)
-    # return remote_file
-    # before rsyncing you have to mirror it on this computer
-    import pdb; pdb.set_trace()
-    local_mirror = remote_path
-    mirror_file = os.path.join(local_mirror, os.path.basename(local_file))
-    # try:
-    #     os.mkdir(os.path.expanduser(local_mirror))
-    # except FileExistsError:
-    #     pass
-    # shutil.copy2(local_file, os.path.expanduser(local_mirror))
-    # shutil.copy2('/Users/ant12/Documents/git-projects/layout-code/klayout-ipc/examples/box.gds',
-    #              '/Users/ant12/tmp_lyipc/klayout-ipc/examples/')
     rsync(local_file, get_target_hostname() + ':' + remote_path)
-    return remote_file
+    return os.path.join(host_HOME(), remote_file)

From 9020cb691d8461292d243fcfdf6a6aa6c5e61aaf Mon Sep 17 00:00:00 2001
From: Alex Tait <alexander.tait@nist.gov>
Date: Sat, 17 Nov 2018 23:48:08 -0700
Subject: [PATCH 4/6] oh man FINALLY got a remote testing job running and
 simultaneously sending remote files back down to be viewed automatically by
 lyipc

---
 README.md                  | 10 +++++++---
 lyipc/client/__init__.py   |  4 ++--
 lyipc/client/general.py    |  1 -
 lyipc/client/remotehost.py | 11 ++++++++---
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index d428935..7c0e478 100644
--- a/README.md
+++ b/README.md
@@ -129,7 +129,7 @@ Following are the steps to enabling this
 #### Network IPC (Done)
 Run a server on one computer. Configure something in lyipc in the second computer. Send lyipc commands. At first, do load with the gds already on the first computer. Next, combine with rsync and gds on local computer with client.load
 
-#### stdout piping
+#### stdout piping (don't bother)
 Test script will look something like
 ```python
 with redirect_stdout():
@@ -139,7 +139,7 @@ This script should be initiated by the laptop but run on the HPC.
 
 I got this working, but its not live.
 
-#### remote build
+#### remote build (Done)
 1. [laptop user] lyipc-job script.py
 1. [laptop] rsync script.py
 1. [HPC] python script.py
@@ -147,10 +147,14 @@ I got this working, but its not live.
 
 Should this use container functions?
 
-#### file transfer and IPC and lytest
+#### file transfer and IPC and lytest (done)
 Set some configuration of lytest, which sets some configuration of lyipc. Run `lytest diff file1.gds file2.gds`. These files are shipped to remote. XOR is run there. Error is detected and sent back to the klayout GUI of the first computer. This will involve actual file transfer.
 
+Edit: this did not set anything in lytest. It was a matter of lyipc:`set_target_hostname` and the HPC using `ship_file` to get things back down.
+
+
 #### script building
+Not yet... just need to sync the ref_layouts.
 lytest not only sends a ref_layout but also a script. This scripted layout is built remotely. The XOR is done remotely.
 
 #### tiling
diff --git a/lyipc/client/__init__.py b/lyipc/client/__init__.py
index 37e59e9..d97a060 100644
--- a/lyipc/client/__init__.py
+++ b/lyipc/client/__init__.py
@@ -37,7 +37,7 @@ def send(message='ping 1234', port=lyipc.PORT):
     psock = QTcpSocket()
     if not isGSI():
         payload = payload.encode()
-    psock.connectToHost(get_target_hostname(), port)
+    psock.connectToHost(get_target_hostname(incl_user=False), port)
     if psock.waitForConnected():
         psock.write(payload)
         if psock.waitForReadyRead(3000):
@@ -48,7 +48,7 @@ def send(message='ping 1234', port=lyipc.PORT):
         else:
             raise Exception('Not acknowledged')
     else:
-        print('Connection Fail! (tried {}:{})'.format(get_target_hostname(), port))
+        print('Connection Fail! (tried {}:{})'.format(get_target_hostname(incl_user=False), port))
     # psock.close()
 
 
diff --git a/lyipc/client/general.py b/lyipc/client/general.py
index 8d77b33..833a4e7 100644
--- a/lyipc/client/general.py
+++ b/lyipc/client/general.py
@@ -27,7 +27,6 @@ def load(filename, mode=None):
     tokens = ['load', filename]
     if mode is not None:
         tokens.append(str(mode))
-    target
     send(' '.join(tokens))
 
 
diff --git a/lyipc/client/remotehost.py b/lyipc/client/remotehost.py
index a470bd1..1f3a49d 100644
--- a/lyipc/client/remotehost.py
+++ b/lyipc/client/remotehost.py
@@ -8,6 +8,8 @@ def set_target_hostname(hostalias, persist=False):
     ''' if it is a remote, you must have already set up an RSA key and alias in your ~/.ssh/config file.
         On that computer, this computer's RSA key needs to be in ~/.ssh/authorized_keys.
         Instructions: https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys--2
+
+        Example: atait@tait-computer
     '''
     global target_host
     if hostalias == 'localhost':
@@ -17,11 +19,14 @@ def set_target_hostname(hostalias, persist=False):
         os.environ['LYIPCTARGET'] = target_host
 
 
-def get_target_hostname():
+def get_target_hostname(incl_user=True):
     try:
-        return os.environ['LYIPCTARGET']
+        host = os.environ['LYIPCTARGET']
     except KeyError:
-        return target_host
+        host = target_host
+    if not incl_user:
+        host = host.split('@')[-1]
+    return host
 
 
 set_target_hostname('localhost')

From 40d424fb024244af6c135c39117c87bb93c646a1 Mon Sep 17 00:00:00 2001
From: Alex Tait <alexander.tait@nist.gov>
Date: Mon, 19 Nov 2018 11:38:49 -0700
Subject: [PATCH 5/6] changed precedence of host assignment. python
 process-wide outranks shell process-wide, if set

---
 lyipc/client/remotehost.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/lyipc/client/remotehost.py b/lyipc/client/remotehost.py
index 1f3a49d..bdba343 100644
--- a/lyipc/client/remotehost.py
+++ b/lyipc/client/remotehost.py
@@ -10,26 +10,33 @@ def set_target_hostname(hostalias, persist=False):
         Instructions: https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys--2
 
         Example: atait@tait-computer
+
+        persist means through the terminal session. It sets environment variable
+        not persist means through this python session. It doesn't last as long. It takes precedence.
     '''
     global target_host
     if hostalias == 'localhost':
         hostalias = socket.gethostbyname(hostalias)
-    target_host = hostalias
     if persist:
         os.environ['LYIPCTARGET'] = target_host
+    else:
+        target_host = hostalias
 
 
 def get_target_hostname(incl_user=True):
-    try:
-        host = os.environ['LYIPCTARGET']
-    except KeyError:
+    if target_host is not None:
         host = target_host
+    else:
+        try:
+            host = os.environ['LYIPCTARGET']
+        except KeyError:
+            return socket.gethostbyname('localhost')
     if not incl_user:
         host = host.split('@')[-1]
     return host
 
 
-set_target_hostname('localhost')
+# set_target_hostname('localhost')
 
 
 def is_host_remote():
@@ -53,7 +60,8 @@ def call_report(command, verbose=True):
 def call_ssh(command):
     # command[0] = '"' + command[0]
     # command[-1] = command[-1] + '"'
-    ssh_command = ['ssh', '-t', get_target_hostname()] + command
+    ssh_command = ['ssh', '-qt', get_target_hostname()]  # q silences welcome banner, t retains text coloring
+    ssh_command += command
     return call_report(ssh_command)
 
 

From 2469d7b7c6f8a7e87d01899d41171ce285816e7b Mon Sep 17 00:00:00 2001
From: Alex Tait <alexander.tait@nist.gov>
Date: Sat, 1 Dec 2018 01:55:19 -0700
Subject: [PATCH 6/6] some notes on remote jobs

---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7c0e478..2a6e76b 100644
--- a/README.md
+++ b/README.md
@@ -152,15 +152,18 @@ Set some configuration of lytest, which sets some configuration of lyipc. Run `l
 
 Edit: this did not set anything in lytest. It was a matter of lyipc:`set_target_hostname` and the HPC using `ship_file` to get things back down.
 
+Notes: to send a file for testing, to call commands and get printouts, to rsync (either direction) -- you need a one-way RSA authorization. If you want to run remote tests that pop up in the local GUI, that currently *requires a two-way RSA authorization*. When the HPC is running, its lytest has the ball (kind of). It decides when to send a pair of files to lyipc. Then lyipc notices that it has to ship those files remotely, requiring rsync. Huh, what if the QTcpSocket in lyipc could send a notice back down that said: rsync this thing from me; it is ready.
+
 
 #### script building
 Not yet... just need to sync the ref_layouts.
 lytest not only sends a ref_layout but also a script. This scripted layout is built remotely. The XOR is done remotely.
 
+
 #### tiling
 To take full advantage, we eventually want to distribute tiles over cores. At first, we will get good results from xdist alone... when it comes to testing, but not dataprep.
 
-
+Remember to pip install pytest-xdist. The error message is not helpful.
 
 #### Author: Alex Tait, June 2018
 #### National Institute of Standards and Technology, Boulder, CO, USA