# Keeping an eye on a file where data lines are added.
# Uses bash shell command "tail -f -n 1"
# The file can be a log file, stock markets, electrity spot prices etc.
# The generator watch_file keeps reading the last line from file as soon as it appears.
# 
# The rest is for testing.
#
import numpy as np
import time
import threading 
import subprocess
import shlex


def watch_file(filename,*args,delay=0.1):
    print('watch file',filename)
    cmd = shlex.split('tail -f -n 1 '+filename)
    proc = subprocess.Popen(cmd,stdout=subprocess.PIPE) 
    while True:
        line = proc.stdout.readline().decode('ascii')  
        if not line:
            print(line)
            time.sleep(delay)
            continue
        yield line
        
def analyze_data(filename):
    ave = 0.0
    N = 0
    for line in watch_file(filename,delay=1.5):
        d = float(line) # or float(line.split()[0])
        ave += d
        N +=1
        print(f'got {d:10.5f} average {ave/N:10.5f} number of data {N}')

def add_new_data(filename):
        for _ in range(10):
            # append data to file
            with open(filename,'a') as f:
                f.write(f'{np.random.random():<10.5f}\n')
            time.sleep(1)
        
if __name__=='__main__':

    filename = 'random.dat'
    t1 = threading.Thread(target=add_new_data,args=(filename,)).start()
    #t2 = threading.Thread(target=analyze_data,args=(filename,)).start()
    analyze_data(filename)