Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HPCC-32847 Require permission to include logs in ZAP #19389

Open
wants to merge 1 commit into
base: candidate-9.10.x
Choose a base branch
from
+68 −34
Open

HPCC-32847 Require permission to include logs in ZAP #19389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 63 additions & 29 deletions esp/services/ws_workunits/ws_workunitsHelpers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@

namespace ws_workunits {

#ifdef _CONTAINERIZED
static const char* LOG_ACCESS_FEATURE = "WsLogAccess";
rpastrana marked this conversation as resolved.
Show resolved Hide resolved
#else
static const char* LOG_ACCESS_FEATURE = "ClusterTopologyAccess";
#endif

const char * const timerFilterText = "measure[time],source[global],depth[1,]"; // Does not include hthor subgraph timings
const char* zipFolder = "tempzipfiles" PATHSEPSTR;

Expand Down Expand Up @@ -3992,13 +3998,12 @@ void CWsWuFileHelper::cleanFolder(IFile* folder, bool removeFolder)
}

#ifndef _CONTAINERIZED
void CWsWuFileHelper::createProcessLogfile(IConstWorkUnit* cwu, WsWuInfo& winfo, const char* process, const char* path)
void CWsWuFileHelper::createProcessLogfile(IConstWorkUnit* cwu, WsWuInfo& winfo, const char* process, const char* path, bool hasLogsAccess)
{
BoolHash uniqueProcesses;
Owned<IPropertyTreeIterator> procs = cwu->getProcesses(process, NULL);
ForEach (*procs)
{
StringBuffer logSpec;
IPropertyTree& proc = procs->query();
const char* processName = proc.queryName();
if (isEmpty(processName))
Expand All @@ -4022,26 +4027,32 @@ void CWsWuFileHelper::createProcessLogfile(IConstWorkUnit* cwu, WsWuInfo& winfo,
pid.appendf("%d", proc.getPropInt("@pid"));

fileName.append("_eclagent.log");
if (!hasLogsAccess)
rpastrana marked this conversation as resolved.
Show resolved Hide resolved
throw makeStringExceptionV(ECLWATCH_ACCESS_TO_FILE_DENIED, "Access to log files is denied. You are missing permission %s:READ - check with your system admin.", LOG_ACCESS_FEATURE);

winfo.getWorkunitEclAgentLog(processName, nullptr, pid.str(), mb, fileName.str());
}
else if (strieq(process, "Thor"))
{
fileName.append("_thormaster.log");
if (!hasLogsAccess)
throw makeStringExceptionV(ECLWATCH_ACCESS_TO_FILE_DENIED, "Access to log files is denied. You are missing permission %s:READ - check with your system admin.", LOG_ACCESS_FEATURE);

winfo.getWorkunitThorMasterLog(processName, nullptr, mb, fileName.str());
}
}
catch(IException* e)
{
StringBuffer s;
e->errorMessage(s);
IERRLOG("Error accessing Process Log file %s: %s", logSpec.str(), s.str());
IERRLOG("Error accessing Process Log file %s: %s", processName, s.str());
writeToFile(fileName.str(), s.length(), s.str());
e->Release();
}
}
}

void CWsWuFileHelper::createThorSlaveLogfile(IConstWorkUnit* cwu, WsWuInfo& winfo, const char* path)
void CWsWuFileHelper::createThorSlaveLogfile(IConstWorkUnit* cwu, WsWuInfo& winfo, const char* path, bool hasLogsAccess)
{
if (cwu->getWuidVersion() == 0)
return;
Expand All @@ -4056,6 +4067,18 @@ void CWsWuFileHelper::createThorSlaveLogfile(IConstWorkUnit* cwu, WsWuInfo& winf
return;
}

// Since there is not already a handler inserting exceptions into the thor slave log,
// make a simple case here to insert a message about permission failure and write it to
// a file that could be recognized as a thor slave log.
if (!hasLogsAccess)
{
StringBuffer msg;
msg.appendf("Access to log files is denied. You are missing permission %s:READ - check with your system admin.", LOG_ACCESS_FEATURE);
VStringBuffer fileName("%s%cthorslave.log", path, PATHSEPCHAR);
writeToFile(fileName.str(), msg.length(), msg.str());
return;
}

Owned<IThreadFactory> threadFactory = new CGetThorSlaveLogToFileThreadFactory();
Owned<IThreadPool> threadPool = createThreadPool("WsWuFileHelper GetThorSlaveLogToFile Thread Pool",
threadFactory, false, nullptr, thorSlaveLogThreadPoolSize, INFINITE);
Expand Down Expand Up @@ -4097,8 +4120,7 @@ void CWsWuFileHelper::createThorSlaveLogfile(IConstWorkUnit* cwu, WsWuInfo& winf
}
#endif

void CWsWuFileHelper::createZAPInfoFile(const char* url, const char* esp, const char* thor, const char* problemDesc,
const char* whatChanged, const char* timing, IConstWorkUnit* cwu, const char* tempDirName)
void CWsWuFileHelper::createZAPInfoFile(CWsWuZAPInfoReq &request, IConstWorkUnit* cwu, const char* tempDirName)
{
VStringBuffer fileName("%s%c%s.txt", tempDirName, PATHSEPCHAR, cwu->queryWuid());
Owned<IFileIOStream> outFile = createBufferedIOStreamFromFile(fileName.str(), IFOcreate);
Expand All @@ -4110,11 +4132,11 @@ void CWsWuFileHelper::createZAPInfoFile(const char* url, const char* esp, const
sb.append("User: ").append(cwu->queryUser()).append("\r\n");
sb.append("Build Version:").append(getBuildVersion()).append("\r\n");
sb.append("Cluster: ").append(cwu->queryClusterName()).append("\r\n");
sb.append("ESP: ").append(esp).append("\r\n");
if (!isEmptyString(url))
sb.append("URL: ").append(url).append("\r\n");
if (!isEmptyString(thor))
sb.append("Thor: ").append(thor).append("\r\n");
sb.append("ESP: ").append(request.esp).append("\r\n");
if (!isEmptyString(request.url))
sb.append("URL: ").append(request.url).append("\r\n");
if (!isEmptyString(request.thor))
sb.append("Thor: ").append(request.thor).append("\r\n");
outFile->write(sb.length(), sb.str());

//Exceptions/Warnings/Info
Expand Down Expand Up @@ -4148,9 +4170,9 @@ void CWsWuFileHelper::createZAPInfoFile(const char* url, const char* esp, const
}

//User provided Information
writeZAPWUInfoToIOStream(outFile, "Problem: ", problemDesc);
writeZAPWUInfoToIOStream(outFile, "What Changed: ", whatChanged);
writeZAPWUInfoToIOStream(outFile, "Timing: ", timing);
writeZAPWUInfoToIOStream(outFile, "Problem: ", request.problemDesc);
writeZAPWUInfoToIOStream(outFile, "What Changed: ", request.whatChanged);
writeZAPWUInfoToIOStream(outFile, "Timing: ", request.whereSlow);
}

void CWsWuFileHelper::writeZAPWUInfoToIOStream(IFileIOStream* outFile, const char* name, SCMStringBuffer& value)
Expand Down Expand Up @@ -4282,6 +4304,8 @@ void CWsWuFileHelper::readWULogToFile(const char *logFileName, WsWuInfo &winfo,
{
try
{
if (!zapLogFilterOptions.hasLogsAccess)
throw makeStringExceptionV(ECLWATCH_ACCESS_TO_FILE_DENIED, "Access to log files is denied. You are missing permission %s:READ - check with your system admin.", LOG_ACCESS_FEATURE);
winfo.readWorkunitComponentLogs(logFileName, zapLogFilterOptions);
}
catch(IException* e)
Expand Down Expand Up @@ -4522,36 +4546,36 @@ int CWsWuFileHelper::zipAFolder(const char* folder, bool gzip, const char* zipFi
void CWsWuFileHelper::createWUZAPFile(IEspContext& context, IConstWorkUnit* cwu, CWsWuZAPInfoReq& request,
const char* tempDirName, StringBuffer& zapFileName, StringBuffer& zipFileNameWithFullPath, unsigned _thorSlaveLogThreadPoolSize)
{
request.hasLogsAccess = context.validateFeatureAccess(LOG_ACCESS_FEATURE, SecAccess_Read, false);
setZAPReportName(request.zapFileName, cwu->queryWuid(), zapFileName);

zipFileNameWithFullPath.set(tempDirName).append(PATHSEPCHAR).append("zapreport.zip");
thorSlaveLogThreadPoolSize = _thorSlaveLogThreadPoolSize;

//create WU ZAP files
createZAPInfoFile(request.url.str(), request.esp.str(), request.thor.str(), request.problemDesc.str(), request.whatChanged.str(),
request.whereSlow.str(), cwu, tempDirName);
//create WU ZAP files
createZAPInfoFile(request, cwu, tempDirName);
createZAPECLQueryArchiveFiles(cwu, tempDirName);

WsWuInfo winfo(context, cwu);
createZAPWUXMLFile(winfo, tempDirName);
createZAPWUGraphProgressFile(request.wuid.str(), tempDirName);

StringArray localFiles;
createZAPWUQueryAssociatedFiles(cwu, tempDirName, localFiles);
createZAPWUQueryAssociatedFiles(cwu, tempDirName, localFiles, request.hasLogsAccess);
#ifndef _CONTAINERIZED
createProcessLogfile(cwu, winfo, "EclAgent", tempDirName);
createProcessLogfile(cwu, winfo, "Thor", tempDirName);
if (request.includeThorSlaveLog.isEmpty() || strieq(request.includeThorSlaveLog.str(), "on"))
createThorSlaveLogfile(cwu, winfo, tempDirName);
createProcessLogfile(cwu, winfo, "EclAgent", tempDirName, request.hasLogsAccess);
createProcessLogfile(cwu, winfo, "Thor", tempDirName, request.hasLogsAccess);
if (request.includeThorSlaveLog.isEmpty() || strieq(request.includeThorSlaveLog.str(), "on"))
createThorSlaveLogfile(cwu, winfo, tempDirName, request.hasLogsAccess);
#else
readWULogToFiles(cwu, winfo, tempDirName, request);
readWULogToFiles(cwu, winfo, tempDirName, request);
#endif

//Write out to ZIP file
zipAllZAPFiles(tempDirName, localFiles, request.password, zipFileNameWithFullPath);
}

void CWsWuFileHelper::createZAPWUQueryAssociatedFiles(IConstWorkUnit* cwu, const char* tempDirName, StringArray& localFiles)
void CWsWuFileHelper::createZAPWUQueryAssociatedFiles(IConstWorkUnit* cwu, const char* tempDirName, StringArray& localFiles, bool hasLogsAccess)
{
Owned<IConstWUQuery> query = cwu->getQuery();
if (!query)
Expand All @@ -4571,6 +4595,21 @@ void CWsWuFileHelper::createZAPWUQueryAssociatedFiles(IConstWorkUnit* cwu, const
RemoteFilename rfn;
SocketEndpoint ep(ip.str());
rfn.setPath(ep, name.str());
StringBuffer fileName(name.str());
getFileNameOnly(fileName, false);

VStringBuffer outFileName("%s%c%s", tempDirName, PATHSEPCHAR, fileName.str());

// Since users are only accustomed to seeing error messages replacing log file contents,
// special case an access denied error message for the one log file processed here.
if (!hasLogsAccess && endsWith(name.str(), ".eclcc.log"))
{
StringBuffer msg;
msg.appendf("Access to log files is denied. You are missing permission %s:READ - check with your system admin.", LOG_ACCESS_FEATURE);
writeToFile(outFileName.str(), msg.length(), msg.str());
continue;
}

if (rfn.isLocal())
{
localFiles.append(name.str());
Expand All @@ -4584,11 +4623,6 @@ void CWsWuFileHelper::createZAPWUQueryAssociatedFiles(IConstWorkUnit* cwu, const
continue;
}

StringBuffer fileName(name.str());
getFileNameOnly(fileName, false);

VStringBuffer outFileName("%s%c%s", tempDirName, PATHSEPCHAR, fileName.str());

OwnedIFile outFile = createIFile(outFileName);
if (!outFile)
{
Expand Down
10 changes: 5 additions & 5 deletions esp/services/ws_workunits/ws_workunitsHelpers.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,7 @@ struct CWsWuZAPInfoReq
bool sendEmail, attachZAPReportToEmail;
bool includeRelatedLogs = true, includePerComponentLogs = false;
unsigned maxAttachmentSize, port;
bool hasLogsAccess = false;

WUComponentLogOptions logFilter;

Expand Down Expand Up @@ -888,15 +889,14 @@ class CWsWuFileHelper
IFile *createWorkingFolder(IEspContext &context, const char *wuid, const char *namePrefix,
StringBuffer &namePrefixStr, StringBuffer &folderName);

void createZAPInfoFile(const char *url, const char *espIP, const char *thorIP, const char *problemDesc,
const char *whatChanged, const char *timing, IConstWorkUnit *cwu, const char *pathNameStr);
void createZAPInfoFile(CWsWuZAPInfoReq &request, IConstWorkUnit *cwu, const char *pathNameStr);
void createZAPWUXMLFile(WsWuInfo &winfo, const char *pathNameStr);
void createZAPECLQueryArchiveFiles(IConstWorkUnit *cwu, const char *pathNameStr);
void createZAPWUQueryAssociatedFiles(IConstWorkUnit *cwu, const char *pathToCreate, StringArray &localFiles);
void createZAPWUQueryAssociatedFiles(IConstWorkUnit *cwu, const char *pathToCreate, StringArray &localFiles, bool hasLogsAccess);
void createZAPWUGraphProgressFile(const char *wuid, const char *pathNameStr);
#ifndef _CONTAINERIZED
void createProcessLogfile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *process, const char *path);
void createThorSlaveLogfile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path);
void createProcessLogfile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *process, const char *path, bool hasLogsAccess);
void createThorSlaveLogfile(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, bool hasLogsAccess);
#endif
LogAccessLogFormat getComponentLogFormatFromLogName(const char *log);
void readWULogToFiles(IConstWorkUnit *cwu, WsWuInfo &winfo, const char *path, CWsWuZAPInfoReq &zapLogFilterOptions);
Expand Down
Loading