Method for real time obtaining Nginx log corresponding to task request by Python

Requirements description

In the process of project requirements testing, it is necessary to send some use case requests to Nginx server, and then check the corresponding Nginx log to judge whether there are characteristic contents and whether the task is successfully executed. In order to improve efficiency, this process needs to be automated.

Practical environment

Python 3.6.5

Code design and implementation

#!/usr/bin/env python
# -*- coding:utf-8 -*-


'''
@CreateTime: 2021/06/26 9:05
@Author : shouke
'''


import time
import threading
import subprocess
from collections import deque


def collect_nginx_log():
    global nginx_log_queue
    global is_tasks_compete
    global task_status


    args = 'tail -0f /usr/local/openresty/nginx/logs/access.log'
    while task_status != 'req_log_got':
        with subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, universal_newlines = True) as proc:
            log_for_req = ''
            outs, errs = '', ''

            try:
                outs, errs = proc.communicate(timeout=2)
            except subprocess.TimeoutExpired:
                print(' Get nginx Log timed out, retrying ')
                proc.kill()
                try:
                    outs, errs = proc.communicate(timeout=5)
                except subprocess.TimeoutExpired:
                    print(' Get nginx Log timeout, timeout again, stop retry ')
                    break
            finally:
                for line in outs.split('\n'):
                    flag = '\"client_ip\":\"10.118.0.77\"' #  Characteristic 
                    if flag in line: #  Find logs that contain feature content 
                        log_for_req += line

                if task_status == 'req_finished':
                    nginx_log_queue.append(log_for_req)
                    task_status = 'req_log_got'



def run_tasks(task_list):
    '''
     Run a task 
    :param task_list  Task list 
    '''

    global nginx_log_queue
    global is_tasks_compete
    global task_status


    for task in task_list:
        thread = threading.Thread(target=collect_nginx_log,
                                    name="collect_nginx_log")
        thread.start()
        time.sleep(1) #  Prepare the log collection thread before executing the task 

        print(' Ongoing mission: %s' % task.get('name'))

        #  Execute Nginx Task request 
        # ...

        task_status = 'req_finished'
        time_to_wait = 0.1
        while task_status != 'req_log_got': #  Request-triggered nginx Log collection incomplete 
            time.sleep(time_to_wait)
            time_to_wait += 0.01
        else:#  Object triggered by a use case request nginx Journal 
            if nginx_log_queue:
                nginx_log = nginx_log_queue.popleft()
                task_status = 'req_ready'
                #  Parse log 
                # do something here
                # ...
            else:
                print(' The queue for storing the request log is empty ')
                # do something here
                # ...


if __name__ == '__main__':
    nginx_log_queue = deque()
    is_tasks_compete = False #  Are all tasks completed 

    task_status = 'req_ready' # req_ready , req_finished , req_log_got  #  Object that stores the task that executes the secondary task 1 Some states 
    print('########################### Task commencement ###########################')

    tast_list = [{'name':'test_task', 'other':'...'}]
    run_tasks(tast_list)

    is_tasks_compete = True

    current_active_thread_num = len(threading.enumerate())
    while current_active_thread_num != 1:
        time.sleep(2)
        current_active_thread_num = len(threading.enumerate())
    print('########################### Task completion ###########################')

Note:

1. Why is the above code not in place in one step, directly tail -0f /usr/local/openresty/nginx/logs/access.log | grep "特征内容" What about? This is because you can't get the log to Nginx by doing so

2. It is found in practice that the first execution proc.communicate(timeout=2) When getting the log, it is always impossible to get it, it will time out, it needs to get it twice, and timeout Setting is too small (in practice, I tried to set it to 1秒 ) will also result in the Nginx log not being retrieved on the second execution.