
    dF              
          d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZmZmZ d dlmZmZmZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ  e       Z e       Z e       Zej@                  jC                  ed      Z"ej@                  jC                  e"d      Z#ej@                  jC                  e"d      Z$ej@                  jC                  e"d      Z%dZ&dZ'dZ(eej@                  jC                  ed      ej@                  jC                  edd      ej@                  jC                  edd      ej@                  jC                  eddd      djS                  e      gZ*dZ+dZ, ejZ                  e.      Z/ G d de0      Z1y)    N)datetime)heappushheappop)	CpuCgroupAGENT_LOG_COLLECTORMemoryCgroup)get_lib_dirget_ext_log_dirget_agent_log_file)0initialize_event_logger_vminfo_common_parametersustr)MANIFEST_NORMALMANIFEST_FULL)GoalStateProperties)get_protocol_utillogcollector	truncatedresults.txtzlogs.zipzcollect-logs.scope      zwaagent_status.jsonhistoryz*.zip*z{0}.*i  i  `	c                   L   e Zd ZdZddZed        Zed        Zed        Zed        Z	ed	        Z
ed
        Zeej                  dfd       Zed        Zd Zed        Zed        Zed        Zed        Zed        Zed        Zd Zed        Zd Zd Zd Zd Zd Zy)LogCollector
truncated_FNc                     || _         |rt        nt        | _        | j	                         | _        | j                          | j                          | j                          | j                  ||      | _
        y N)_is_full_moder   r   	_manifest_expand_must_collect_files_must_collect_files_create_base_dirs_set_logger_initialize_telemetry_set_resource_usage_cgroupscgroups)selfis_full_modecpu_cgroup_pathmemory_cgroup_paths       E/usr/lib/python3/dist-packages/azurelinuxagent/common/logcollector.py__init__zLogCollector.__init__J   s_    )*6O#'#B#B#D  ""$77I[\    c                 n    t         j                  j                  |       st        j                  |        y y r   )ospathisdirmakedirs)dirnames    r,   _mkdirzLogCollector._mkdirS   s#    ww}}W%KK  &r.   c                     t        | d      5 }|j                  dj                  d             d d d        y # 1 sw Y   y xY w)Nwb utf-8)openwriteencode)filepathout_files     r,   _reset_filezLogCollector._reset_fileX   s8    (D! 	/XNN299W-.	/ 	/ 	/s	   !7A c                  h    t         j                  t               t         j                  t               y r   )r   r5   _LOG_COLLECTOR_DIR_TRUNCATED_FILES_DIR r.   r,   r#   zLogCollector._create_base_dirs]   s     ./01r.   c                  $   t        j                  t        d      } t        j                  dd      }t        j
                  |_        | j                  |       t        j                  |        t        j                  t         j                         y )Nr9   )encodingz%%(asctime)s %(levelname)s %(message)s%Y-%m-%dT%H:%M:%SZ)fmtdatefmt)loggingFileHandlerOUTPUT_RESULTS_FILE_PATH	Formattertimegmtime	convertersetFormatter_LOGGER
addHandlersetLevelINFO)
_f_handler	_f_formats     r,   r$   zLogCollector._set_loggerb   sf    (()AGT
%%*Q.CE	"kk		*:&&r.   c                    t        t        |       }dj                  |      }t        j	                  |       |j                          t        t        |      }dj                  |      }t        j	                  |       ||gS )NzStarted tracking cpu cgroup {0}z"Started tracking memory cgroup {0})r   r   formatrQ   infoinitialize_cpu_usager   )r*   r+   
cpu_cgroupmsgmemory_cgroups        r,   r&   z(LogCollector._set_resource_usage_cgroupsl   sl    2OD
/66zBS'')$%8:LM299-HSM**r.   c                      t               j                  d      } | j                  j                  t        j
                  t        j                  z         t        |        y )NF)init_goal_state)goal_state_properties)r   get_protocolclientreset_goal_stater   
RoleConfig
HostingEnvr   )protocols    r,   r%   z"LogCollector._initialize_telemetryw   sH    $&33E3J((?R?]?]`s`~`~?~(8Br.   c                 2     fd}d }	 t        j                   |t         j                  d      }|j                         \  }}|j                  }|dk7  r@ ||      }
 ||      }dj                   |       ||
|      }	t        j                  |	       y|r4d	j                   |        ||            }t        j                  |       yy# t
        $ r@}dj                   |       t        |            }	t        j                  |	       Y d}~yd}~ww xY w)
a  
        Runs a shell command in a subprocess, logs any errors to the log file, enables changing the stdout stream,
        and logs the output of the command to the log file if indicated by the `log_output` parameter.
        :param command: Shell command to run
        :param stdout: Where to write the output of the command
        :param log_output: If true, log the command output to the log file
        c                 J    t        | t              rdj                  |       S S )N )
isinstancelistjoin)cmdcommands    r,   format_commandz7LogCollector._run_shell_command.<locals>.format_command   s    $.sD$9388C=FwFr.   c                     t        | dd      S )Nr9   backslashreplace)rE   errorsr   )outputs    r,   _encode_command_outputz?LogCollector._run_shell_command.<locals>._encode_command_output   s    9KLLr.   F)stdoutstderrshellz0Command [{0}] raised unexpected exception: [{1}]Nr   z?Command: [{0}], return code: [{1}], stdout: [{2}] stderr: [{3}]zOutput of command [{0}]:
{1})
subprocessPopenPIPEcommunicate
returncode	ExceptionrX   r   rQ   errorrY   )rn   ru   
log_outputro   rt   processrv   return_codee	error_msgencoded_stdoutencoded_stderrr\   s   `            r,   _run_shell_commandzLogCollector._run_shell_command~   s   	G	M	 &&wvjoo]bcG$002NFF!,,K !3F;N3F;NY``aopwaxalaoaoqI MM)$1889PRhioRpqCLL   	KRRSabiSjlpqrlstIMM)$	s   AC 	D6DDc                  z    g } t         D ]/  }| j                  t        t        j                  |                   1 | S r   )_MUST_COLLECT_FILESextendsortedglob)manifestr1   s     r,   r!   z'LogCollector._expand_must_collect_files   s8     ' 	5DOOF499T?34	5 r.   c                 6    | j                   j                         S r   )r    
splitlines)r(   s    r,   _read_manifestzLogCollector._read_manifest   s    ~~((**r.   c                 8    t         j                  dd| gd       y )Nlsz-alFT)r   )r   r   )folders    r,   _process_ll_commandz LogCollector._process_ll_command   s    ''vv(>4'Pr.   c                 .    t         j                  |        y r   )rQ   rY   )messages    r,   _process_echo_commandz"LogCollector._process_echo_command   s    Wr.   c                 h    t        j                   |       }|D ]  }t        j                  |        |S r   )r   rQ   rY   )r1   
file_paths	file_paths      r,   _process_copy_commandz"LogCollector._process_copy_command   s0    YYt_
# 	$ILL#	$r.   c                 j   | j                  t              rv| t        t              d  j                  t        j
                  j                        }t        j                  |j                  t        j
                  j                  d      z   }|S | j                  t        j
                  j                        S )N_)

startswithrB   lenlstripr0   r1   sepr   _TRUNCATED_FILE_PREFIXreplace)	file_nameoriginal_file_patharchive_file_names      r,   "_convert_file_name_to_archive_namez/LogCollector._convert_file_name_to_archive_name   s      45!*3/C+D+E!F!M!Mbggkk!Z , C CFXF`F`acahahalalnqFr r$$##BGGKK00r.   c                     t        j                  t              }|D ]`  }t         j                  j	                  t        |      }|| vs,t         j                  j                  |      sLt        j                  |       b y r   )r0   listdirrB   r1   rl   isfileremove)files_to_collecttruncated_filesr   	full_paths       r,   #_remove_uncollected_truncated_filesz0LogCollector._remove_uncollected_truncated_files   s\    
 **%9:( 	)I%99EI 0077>>),IIi(		)r.   c                 Z   t         j                  dt               t         j                  dt               t         j                  dt               g }| D ]U  }|j                  dt              }|j                  dt              }|j                  dt              }|j                  |       W |S )NzUsing %s as $LIB_DIRzUsing %s as $LOG_DIRzUsing %s as $AGENT_LOGz$LIB_DIRz$LOG_DIRz
$AGENT_LOG)rQ   rY   _AGENT_LIB_DIR_EXTENSION_LOG_DIR
_AGENT_LOGr   append)manifest_datanew_manifestlinenew_lines       r,   _expand_parameterszLogCollector._expand_parameters   s    +^<+-?@-z:! 	*D||J?H''
4FGH''jAH)		* r.   c                    t               }| j                         }t        j                  |      }|D ]  }|j	                  d      }t        |      dk7  rD|j                  d      s2t        |j                               dkD  rt        j                  d|       f|\  }}|dk(  r| j                  |       |dk(  r| j                  |       |dk(  s|j                  | j                  |              |S )	N,r   #r   zCouldn't parse "%s"llechocopy)setr   r   r   splitr   r   striprQ   r~   r   r   updater   )r(   r   datamanifest_entriesentrycontentsrn   values           r,   _process_manifest_filez#LogCollector._process_manifest_file   s    5""$'::4@% 	KE {{3'H8}!'',U[[]1Ca1GMM"95A%NGU$((/F"**51F" ''(B(B5(IJ+	K.  r.   c                    	 t         j                  j                  |       d   }|dv rt        j	                  d|        y t         j                  j                  t        | j                  t         j                  j                  d            }t         j                  j                  |      rEt         j                  j                  |       }t         j                  j                  |      }||k  r|S t        |d      5 }t        j                  ddt        t              | g|       d d d        |S # 1 sw Y   |S xY w# t         $ r)}t        j#                  d	t%        |             Y d }~y d }~ww xY w)
N   )z.gzz.zipz.xzzDiscarding large binary file %sr   zw+tailz-c)ru   z!Failed to truncate large file: %s)r0   r1   splitextrQ   warningrl   rB   r   r   existsgetmtimer:   r   r   str_FILE_SIZE_LIMITOSErrorr~   r   )r   exttruncated_file_pathoriginal_file_mtimetruncated_file_mtimefhr   s          r,   _truncate_large_filez!LogCollector._truncate_large_file  s4   	''""9-a0C,, A9M"$'',,/CYEVEVWYW^W^WbWbdgEh"iww~~12&(gg&6&6y&A#')ww'7'78K'L$ ')==.. )40 mB//sCS?TV_0`ik/lm '&m '& 	MM=tAwG	sA   <D; B/D; /D; ;)D.$	D; .D83D; 8D; ;	E-E((E-c                 X    	 | j                   j                  |      S # t        $ r Y yw xY w)Niɚ;)r"   index
ValueError)r(   
file_entrys     r,   _get_file_priorityzLogCollector._get_file_priority%  s0    	++11*== 		s    	))c                 V    g }|D ]!  }| j                  |      }t        |||f       # |S r   )r   r   )r(   	file_listpriority_file_queuer   prioritys        r,   _get_priority_files_listz%LogCollector._get_priority_files_list.  sC     !# 	BJ..z:H(8Z*@A	B #"r.   c                 F   t         j                  d       d}g }|rt        |      d   }t        t        j
                  j                  |      t              }||z   t        kD  rt         j                  d       nt        j
                  j                  |      t        k  r)|j                  |       t         j                  d||       n;| j                  |      }|r(t         j                  d||       |j                  |       ||z  }|rt         j                  d|       |S )Nz1### Preparing list of files to add to archive ###r   r   z(Archive too big, done with adding files.zAdding file %s, size %s bz#Adding truncated file %s, size %s bz!Uncompressed archive size is %s b)rQ   rY   r   minr0   r1   getsizer    _UNCOMPRESSED_ARCHIVE_SIZE_LIMITr   r   r   )r(   r   total_uncompressed_sizefinal_files_to_collectr   	file_sizer   s          r,   _get_final_list_for_archivez(LogCollector._get_final_list_for_archive8  s     	HI"#!#! 34Q7IBGGOOI68HII&25UU JKwwy)-==&--i88)YO&*&?&?	&J#&LL!FH[]fg*112EF#y0## "& 	8:QR%%r.   c                 j    | j                         }| j                  |      }| j                  |      }|S r   )r   r   r   )r(   parsed_file_pathsprioritized_file_pathsr   s       r,    _create_list_of_files_to_collectz-LogCollector._create_list_of_files_to_collectV  s=     !779!%!>!>?P!Q;;<RSr.   c                 d   g }	 | j                          t        j                  t               t	        j
                         }t        j                  d|j                  d             t        j                  d| j                  rdnd       | j                         }t        j                  d       d}	 t        j                  t        dt        j                  	      }|D ]9  }t        j                  |      }|j!                  |j#                  d
      |       ; t$        j&                  j)                  t              }t        j                  d|       t	        j
                         }||z
  }t+        |j,                  dz  dz  dz  |j.                  z   dz  |j0                  dz  z         }	t        j                  d|j                         j                  d             t        j                  d|	       |j!                  t        j#                  d
      d       ||j3                          	 t        | j5                  |       S # ||j3                          w w xY w# t6        $ r5}
dj9                  t;        |
            }t        j=                  |        d}
~
ww xY w# | j5                  |       w xY w)z
        Public method that collects necessary log files in a compressed zip archive.
        :return: Returns the path of the collected compressed archive
        zStarting log collection at %srF   zUsing log collection mode %sfullnormalz#### Creating compressed archive ###Nw)compressionr9   )arcnamez>Successfully compressed files. Compressed archive size is %s b   <   i  g     @@zFinishing log collection at %szElapsed time: %s msr   zFailed to collect logs: {0})r#   r   r?   rK   r   utcnowrQ   rY   strftimer   r   zipfileZipFileCOMPRESSED_ARCHIVE_PATHZIP_DEFLATEDr   r;   r<   r0   r1   r   intdayssecondsmicrosecondscloser   r}   rX   r   r~   )r(   r   
start_timecompressed_archivefile_to_collectr   compressed_archive_sizeend_timeduration
elapsed_msr   r\   s               r,   collect_logs_and_get_archivez)LogCollector.collect_logs_and_get_archivea  sD   
 )	G""$$$%=>!*JLL8*:M:MNb:cdLL74CUCU[cd#DDFLL>?!%/%,__5Lc_f_s_s%t"'7 iO(4(W(WXg(h%&,,_-C-CG-LVg,hi +-''//:Q*R']_vw#??,#j0 8==2#5#:R#?(BRBR#RVZ"Z_g_t_tw}_}!~
=x?P?Y?YZn?op2J?"(()A)H)H)Q[h(i%1&,,.* 445EF &1&,,. 2  	/66tAw?CMM#		 445EFs=   B-I 2E'I I II 	J$0JJJ J/)FNN)__name__
__module____qualname__r   r-   staticmethodr5   r?   r#   r$   r&   r%   rx   rz   r   r!   r   r   r   r   r   r   r   r   r   r   r   r   r   r	  rC   r.   r,   r   r   F   sl   )] ! ! / / 2 2 ' ' + + C C +5??u # #J  + Q Q     1 1 
) 
)   <  :#&<	 0Gr.   r   )2r   rI   r0   rx   rM   r   r   heapqr   r   azurelinuxagent.common.cgroupr   r   r   azurelinuxagent.common.confr	   r
   r   azurelinuxagent.common.eventr   azurelinuxagent.common.futurer   -azurelinuxagent.common.logcollector_manifestsr   r   *azurelinuxagent.common.protocol.goal_stater   $azurelinuxagent.common.protocol.utilr   r   r   r   r1   rl   rA   rB   rK   r   CGROUPS_UNITGRACEFUL_KILL_ERRCODEINVALID_CGROUPS_ERRCODErX   r   r   r   	getLoggerr
  rQ   objectr   rC   r.   r,   <module>r     sY  (   	     # V V X X Y . X K B$& !
WW\\..A ww||$6D 77<<(:MJ '',,'9:F #   GGLL!67GGLLG4GGLL#S#.GGLL#S#s3NN:  $ #4  
'

H
%KG6 KGr.   