DRCEResourceExtractor.cpp

Go to the documentation of this file.

#include <sys/resource.h>
#include <sys/statvfs.h>
#include <utility>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <fstream>
#include <sstream>
#include <dirent.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <time.h>
#include <stdio.h>
#include <stdlib.h>

#include <Poco/TemporaryFile.h>
#include <Poco/Logger.h>
#include <Poco/Message.h>

#include "DRCEAsyncTasksQueue.hpp"
#include "DRCEResourceExtractor.hpp"
#include "DRCEMessageConst.hpp"
#include "DRCEError.hpp"
#include "DRCEReadProcessData.hpp"
#include "DRCEResourceUsage.hpp"

namespace HCE
{
namespace drce
{
//-----------------------------------------------------------------------------
DiskSpaceExtractor::DiskSpaceExtractor(void)
: diskSize(0), diskUsed(0), diskFree(0), diskUsedPercent(0.0), diskFreePercent(0.0), externalDiskSize(0)
{
  try
  {
    update();
  }
  catch(Poco::Exception& e)
  {
    Poco::Logger::root().log(Poco::Message(drce_const::moduleName, "DiskSpaceExtractor "+e.message(), Poco::Message::Priority::PRIO_ERROR));
  }
}
//-----------------------------------------------------------------------------
void DiskSpaceExtractor::setDiskSize(size_t diskSize_)
{
  externalDiskSize = diskSize_;
}
//-----------------------------------------------------------------------------
void DiskSpaceExtractor::update(void) throw (Poco::Exception)
{
  Poco::TemporaryFile temporaryFile;
  temporaryFile.createFile();
  const std::string fileName = temporaryFile.path();

  struct statvfs64 buf={0};

  if (statvfs64(fileName.c_str(), &buf) != 0)
    throw Poco::Exception(std::string("DiskSpaceExtractor::update: ")+strerror(errno));

  size_t blksize = buf.f_bsize;
  size_t blocks = buf.f_blocks;
  size_t freeblks = buf.f_bfree;

  if (externalDiskSize)
    diskSize = externalDiskSize;
  else
    diskSize = blocks * blksize;

  diskFree = freeblks * blksize;
  diskUsed = diskSize - diskFree;

  diskUsedPercent = static_cast<double>(diskUsed*100)/diskSize;
  diskFreePercent = static_cast<double>(diskFree*100)/diskSize;
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
size_t DiskSpaceExtractor::getDiskUsageBytes(pid_t pid)
{
  size_t diskUsage = 0;
  char buf[PATH_MAX] = {0};
  struct dirent* entry = nullptr;

  snprintf(buf, PATH_MAX, "/proc/%i/fd/", pid);

  DIR* dir = opendir(buf);
  if (dir)
  {
    while ((entry = readdir(dir)) != nullptr)
    {
      if (entry->d_name != std::string(".") && entry->d_name !=  std::string(".."))
      {
        diskUsage += getFileSize(std::string(buf)+"/"+entry->d_name);
      }
    }
    closedir(dir);
  }
  return diskUsage;
}
//-----------------------------------------------------------------------------
size_t DiskSpaceExtractor::getFileSize(const std::string& filename)
{
  size_t result = 0;
  struct stat buff = {0};

  if (stat (filename.c_str(), &buff) == 0)
  {
    result = buff.st_size;
  }
  return result;
}
//-----------------------------------------------------------------------------
size_t DiskSpaceExtractor::getDiskUsage(const std::vector<pid_t>& ids)
{
  size_t diskUsage = 0;

  for (size_t i=0;i<ids.size();++i)
  {
    diskUsage += getDiskUsageBytes(ids[i]);
  }
  return diskUsage;
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
void CpuUsageExtractor::getCpuData(pid_t pid, std::pair<unsigned long int, unsigned long int>& p)
{
  const unsigned int count = 14;
  char path[PATH_MAX]={0};
  char pId[PATH_MAX]={0};
  snprintf(path, PATH_MAX, "%s%d%s", "/proc/", pid, "/stat");

  FILE* fp = fopen(path, "r");
  if (fp)
  {
    for(unsigned int i = 1; i < count; ++i)
    {
      fscanf(fp, "%s", pId);
    }

    fscanf(fp, "%ld%ld", &p.first, &p.second);
    fclose(fp);
  }
}
//-----------------------------------------------------------------------------
/*
 * calculates the elapsed CPU usage for list of processes
 */
double CpuUsageExtractor::getCpuUsage(const std::vector<pid_t>& ids)
{
  const double defaultUsage = 100.0;
  const unsigned int SLPTIME = 200000;

  std::vector<std::pair<unsigned long int, unsigned long int> > cpuPrevData, cpuData;

  for (size_t i=0;i<ids.size();++i)
  {
    std::pair<unsigned long int, unsigned long int> data;
    getCpuData(ids[i], data);
    cpuPrevData.push_back(data);
  }

  usleep(SLPTIME);

  for (size_t i=0;i<ids.size();++i)
  {
    std::pair<unsigned long int, unsigned long int> data;
    getCpuData(ids[i], data);
    cpuData.push_back(data);
  }

  unsigned long int q(0), r(0);

  for (size_t i=0;i<cpuData.size();++i)
  {
    if ((cpuData[i].first) && (cpuPrevData[i].first))
      q += abs(cpuData[i].first-cpuPrevData[i].first);

    if ((cpuData[i].second) && (cpuPrevData[i].second))
      r += abs(cpuData[i].second-cpuPrevData[i].second);
  }

  double result = defaultUsage;
  long int frec = sysconf(_SC_CLK_TCK);
  if (frec > 0)
  {
    result = ((q + r)*100.0/((double)0.2*frec));
  }
  return result;
}
//-----------------------------------------------------------------------------
CpuUsageExtractor::CpuUsageExtractor(void)
: cpuUsage(0.0),/* lastBusy(0), lastWork(0),*/ iowait(0.0)/*, lastIOWait(0), lastSum(0)*/
{
  update();
}
//-----------------------------------------------------------------------------
void CpuUsageExtractor::update(void)
{
  memset(&oldCpu, 0, sizeof(oldCpu));
  updateUsage();
  usleep(SLPTIME);
  updateUsage();
}
//-----------------------------------------------------------------------------
void CpuUsageExtractor::updateUsage(void)
{
  std::ifstream ifs("/proc/stat");
  if (ifs.is_open())
  {
    std::string line;
    while(getline(ifs, line))
    {
      if (line.compare(0, 4, "cpu ")==0)
      {
        unsigned long long nice(0), irq(0), softirq(0);

        int items = sscanf(line.c_str(),
                     "cpu %20llu %20llu %20llu %20llu %20llu %20llu %20llu",
                       &newCpu.user, &nice,
                       &newCpu.system,
                       &newCpu.idle,
                       &newCpu.iowait,
                       &irq, &softirq);

        newCpu.user += nice;
        if (items == 4)
          newCpu.iowait = 0;
        if (items == 7)
          newCpu.system += irq + softirq;
        break;
      }
    }
    ifs.close();
  }

  struct cpu_info cpu;

  cpu.user = newCpu.user - oldCpu.user;
  cpu.system = newCpu.system - oldCpu.system;
  cpu.idle = newCpu.idle - oldCpu.idle;
  cpu.iowait = newCpu.iowait - oldCpu.iowait;
  double total = static_cast<double>(cpu.user + cpu.system + cpu.idle + cpu.iowait)/100.0;

  cpuUsage = static_cast<double>(cpu.user/total)+static_cast<double>(cpu.system/total);
  iowait = static_cast<double>(cpu.iowait/total);

  oldCpu = newCpu;
}
//-----------------------------------------------------------------------------
LoadAverageExtractor::LoadAverageExtractor(void)
{
  update();
}
//-----------------------------------------------------------------------------
void LoadAverageExtractor::update(void)
{
  std::ifstream ifs("/proc/loadavg");
  if (ifs.is_open())
  {
    std::string line;
    while(getline(ifs, line))
    {
      sscanf(line.c_str(),"%lf %lf %lf", &loadAverage[0], &loadAverage[1], &loadAverage[2]);
    }
    ifs.close();
  }
}
//-----------------------------------------------------------------------------
double LoadAverageExtractor::getLoadAverage(unsigned int index)
{
  double value = 0.0;
  if (index < SZ)
  {
    value = loadAverage[index];
  }
  return value;
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
MemoryExtractor::MemInfo::MemInfo(void)
: memTotal(0), memFree(0), buffers(0), cached(0), swapTotal(0), swapFree(0), vmallocTotal(0), vmallocUsed(0)
{
}
//-----------------------------------------------------------------------------
MemoryExtractor::MemInfo::MemInfo(MemInfo&& rhs)
: memTotal(0), memFree(0), buffers(0), cached(0), swapTotal(0), swapFree(0), vmallocTotal(0), vmallocUsed(0)
{
  *this = std::forward<MemInfo>(rhs);
}
//-----------------------------------------------------------------------------
MemoryExtractor::MemInfo& MemoryExtractor::MemInfo::operator=(MemInfo&& rhs)
{
  if (this != &rhs)
  {
    memTotal = std::move(rhs.memTotal);
    memFree = std::move(rhs.memFree);
    buffers = std::move(rhs.buffers);
    cached = std::move(rhs.cached);
    swapTotal = std::move(rhs.swapTotal);
    swapFree = std::move(rhs.swapFree);
    vmallocTotal = std::move(rhs.vmallocTotal);
    vmallocUsed = std::move(rhs.vmallocUsed);
  }
  return *this;
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
MemoryExtractor::MemoryExtractor(void)
: vramUsed(0), rramUsed(0), vramFree(0), rramFree(0), vramUsedPercent(0.0), rramUsedPercent(0.0), vramFreePercent(0.0), rramFreePercent(0.0),
  externalVramTotal(0), externalRramTotal(0)
{
  update();
}
//-----------------------------------------------------------------------------
void MemoryExtractor::setVramTotal(size_t vramTotal_)
{
  externalVramTotal = vramTotal_;
}
//-----------------------------------------------------------------------------
void MemoryExtractor::setRramTotal(size_t rramTotal_)
{
  externalRramTotal = rramTotal_;
}
//-----------------------------------------------------------------------------
void MemoryExtractor::update(void)
{
  MemoryExtractor::MemInfo memInfo = getMemoryInfo();

  if (externalRramTotal)
    memInfo.memTotal = externalRramTotal;

  rramFree = (memInfo.memTotal-memInfo.cached-memInfo.buffers)*1024;
  rramUsed = memInfo.memTotal*1024-rramFree;

  size_t vmemTotal = (externalVramTotal)?externalVramTotal:(memInfo.swapTotal+memInfo.memTotal);
  vramFree = (vmemTotal-memInfo.vmallocUsed)*1024;
  vramUsed = memInfo.vmallocUsed*1024;

  rramFreePercent = static_cast<double>(rramFree*100)/(memInfo.memTotal*1024);
  rramUsedPercent = static_cast<double>(rramUsed*100)/(memInfo.memTotal*1024);

  vramFreePercent = static_cast<double>(vramFree*100)/(vmemTotal*1024);
  vramUsedPercent = static_cast<double>(vramUsed*100)/(vmemTotal*1024);

  vramTotal = vmemTotal*1024;
  rramTotal = memInfo.memTotal*1024;
}
//-----------------------------------------------------------------------------
MemoryExtractor::MemInfo MemoryExtractor::getMemoryInfo(void)
{
  MemoryExtractor::MemInfo memInfo;

  std::ifstream ifs("/proc/meminfo");
  if (ifs.is_open())
  {
    std::string line;
    while(getline(ifs, line))
    {
      if (line.find("MemTotal")!=std::string::npos)
        sscanf(line.c_str(), "MemTotal: %zu", &(memInfo.memTotal));

      if (line.find("MemFree")!=std::string::npos)
        sscanf(line.c_str(), "MemFree: %zu", &(memInfo.memFree));

      if (line.find("Buffers")!=std::string::npos)
        sscanf(line.c_str(), "Buffers: %zu", &(memInfo.buffers));

      if (line.find("Cached")!=std::string::npos)
        sscanf(line.c_str(), "Cached: %zu", &(memInfo.cached));

      if (line.find("SwapTotal")!=std::string::npos)
        sscanf(line.c_str(), "SwapTotal: %zu", &(memInfo.swapTotal));

      if (line.find("SwapFree")!=std::string::npos)
        sscanf(line.c_str(), "SwapFree: %zu", &(memInfo.swapFree));

      if (line.find("VmallocTotal")!=std::string::npos)
        sscanf(line.c_str(), "VmallocTotal: %zu", &(memInfo.vmallocTotal));

      if (line.find("VmallocUsed")!=std::string::npos)
        sscanf(line.c_str(), "VmallocUsed: %zu", &(memInfo.vmallocUsed));
    }
    ifs.close();
  }
  return memInfo;
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
ProcessInfoExtractor::FieldsArray::FieldsArray(void)
: cpu(0.0), vram(0), rram(0), threads(0), disk(0)
{
}
//-----------------------------------------------------------------------------
ProcessInfoExtractor::FieldsArray::FieldsArray(const FieldsArray& rhs)
: cpu(0.0), vram(0), rram(0), threads(0), disk(0)
{
  *this = rhs;
}
//-----------------------------------------------------------------------------
ProcessInfoExtractor::FieldsArray::FieldsArray(FieldsArray&& rhs)
: cpu(0.0), vram(0), rram(0), threads(0), disk(0)
{
  *this = std::forward<FieldsArray>(rhs);
}
//-----------------------------------------------------------------------------
ProcessInfoExtractor::FieldsArray& ProcessInfoExtractor::FieldsArray::operator=(const FieldsArray& rhs)
{
  if (this != &rhs)
  {
    cpu = rhs.cpu;
    vram = rhs.vram;
    rram = rhs.rram;
    threads = rhs.threads;
    disk = rhs.disk;
  }
  return *this;
}
//-----------------------------------------------------------------------------
ProcessInfoExtractor::FieldsArray& ProcessInfoExtractor::FieldsArray::operator=(FieldsArray&& rhs)
{
  if (this != &rhs)
  {
    cpu = std::move(rhs.cpu);
    vram = std::move(rhs.vram);
    rram = std::move(rhs.rram);
    threads = std::move(rhs.threads);
    disk = std::move(rhs.disk);
  }
  return *this;
}
//-----------------------------------------------------------------------------
void ProcessInfoExtractor::FieldsArray::clear(void)
{
  cpu = 0.0;
  vram = 0;
  rram = 0;
  threads = 0;
  disk = 0;
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
ProcessInfoExtractor::FieldsArray ProcessInfoExtractor::extract(const AsyncTasks& asyncTask)
{
  std::vector<pid_t> ids = DRCEReadProcessData::getAllProcessIds(asyncTask.pid);

  ProcessInfoExtractor::FieldsArray fieldsArray;

  for (size_t i=0;i<ids.size();++i)
  {
    ProcessData processData = DRCEReadProcessData::getInstance().getProcessData(ids[i]);

    fieldsArray.vram += processData.vram;
    fieldsArray.rram += processData.rram;

    fieldsArray.threads += processData.threads;
  }
  fieldsArray.cpu = CpuUsageExtractor::getCpuUsage(ids);
  fieldsArray.disk = DiskSpaceExtractor::getDiskUsage(ids);

  if (fieldsArray.cpu > 1000 || fieldsArray.threads > 100)
  {
    std::cout << "taskId: " << asyncTask.taskId << " pid: "<< asyncTask.pid << " count: " << ids.size() << std::endl;
    std::cout << "Children: ";
    for (size_t i=0;i<ids.size();++i)
      std::cout << ids[i] << " ";
    std::cout << std::endl;
    std::cout << "cpu: " << fieldsArray.cpu << " vram: " << fieldsArray.vram << " rram: "<< fieldsArray.rram << " threads: " << fieldsArray.threads << std::endl;
  }

  return fieldsArray;
}
//-----------------------------------------------------------------------------
ResourceUsageExtractor* ResourceUsageExtractor::pHandler = nullptr;
Poco::Mutex ResourceUsageExtractor::mutex;
//-----------------------------------------------------------------------------
ResourceUsageExtractor::ResourceUsageExtractor(void)
: cpuUsageExtractor(), diskSpaceExtractor(), memoryExtractor(), loadAverageExtractor(), maxThreadsCount(0), maxProcessesCount(0)
{
}
//-----------------------------------------------------------------------------
ResourceUsageExtractor& ResourceUsageExtractor::getInstance(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  if(!pHandler)
    pHandler = new ResourceUsageExtractor();
  return *pHandler;
}
//-----------------------------------------------------------------------------
void ResourceUsageExtractor::shutdown(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  if(pHandler)
  {
    delete pHandler;
    pHandler=nullptr;
  }
}
//-----------------------------------------------------------------------------
void ResourceUsageExtractor::update(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  cpuUsageExtractor.update();
  diskSpaceExtractor.update();
  memoryExtractor.update();
  loadAverageExtractor.update();
}
//-----------------------------------------------------------------------------
ResourceUsage ResourceUsageExtractor::extract(void) throw (Poco::Exception)
{
  Poco::Mutex::ScopedLock lock(mutex);
  ResourceUsage resourceUsage;

  resourceUsage.cpu = cpuUsageExtractor.getCpuUsage();
  resourceUsage.iowait = cpuUsageExtractor.getIOWait();

  resourceUsage.diskUsed = diskSpaceExtractor.getDiskUsed();
  resourceUsage.diskUsedPercent = diskSpaceExtractor.getDiskUsedPercent();

  resourceUsage.vramUsed = memoryExtractor.getVramUsed();
  resourceUsage.vramUsedPercent = memoryExtractor.getVramUsedPercent();
  resourceUsage.rramUsed = memoryExtractor.getRramUsed();
  resourceUsage.rramUsedPercent = memoryExtractor.getRramUsedPercent();

  resourceUsage.loadAverage = loadAverageExtractor.getLoadAverage(1);

  DRCEReadProcessData::getInstance().getProcessAndThreads(resourceUsage.processes, resourceUsage.threads);

  long sz = sysconf(_SC_PAGESIZE);
  if (sz<0)
    throw Poco::Exception(std::string("ResourceUsageExtractor::extract: ")+strerror(errno));

//  printf("Page size = %ld bytes \n", sz);

  size_t stacksize = 0;
  pthread_attr_t attr;
  pthread_attr_init(&attr);
  pthread_attr_getstacksize(&attr, &stacksize);
//  printf("Thread stack size = %zu bytes \n", stacksize);
  stacksize /= 1024;
//  printf("Thread stack size = %zu Kb \n", stacksize);

  size_t totalRamPages = (memoryExtractor.getRramTotal()/sz);
//  printf("Total ram pages = %zu\n", totalRamPages);

  size_t maxThreads = (maxThreadsCount)?maxThreadsCount:(totalRamPages/(8*stacksize/sz));
//  printf("Max threads = %zu\n", maxThreads);

//  std::cout << "getMaxProcesses(): " << getMaxProcesses() << std::endl;
//  std::cout << "getMaxThreads(): " << getMaxThreads() << std::endl;

  resourceUsage.threadsPercent = static_cast<double>(resourceUsage.threads*100)/maxThreads;

  double numberThreads = static_cast<double>(memoryExtractor.getVramTotal())/(stacksize*1024*1024*1024);
//  printf("Number of threads = %f\n", numberThreads);

  double maxProcesses = (maxProcessesCount)?maxProcessesCount:(static_cast<double>(maxThreads) / numberThreads);
//  printf("Max Processes = %f\n", maxProcesses);

  resourceUsage.processesPercent = static_cast<double>(resourceUsage.processes*100)/maxProcesses;

  //TODO in future, here can be necessary refactoring
  return resourceUsage;
}
//-----------------------------------------------------------------------------
CpuUsageExtractor& ResourceUsageExtractor::getCpuUsageExtractor(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  return cpuUsageExtractor;
}
//-----------------------------------------------------------------------------
DiskSpaceExtractor& ResourceUsageExtractor::getDiskSpaceExtractor(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  return diskSpaceExtractor;
}
//-----------------------------------------------------------------------------
MemoryExtractor& ResourceUsageExtractor::getMemoryExtractor(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  return memoryExtractor;
}
//-----------------------------------------------------------------------------
LoadAverageExtractor& ResourceUsageExtractor::getLoadAverageExtractor(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  return loadAverageExtractor;
}
//-----------------------------------------------------------------------------
void ResourceUsageExtractor::setMaxThreadsCount(size_t maxThreadsCount_)
{
  Poco::Mutex::ScopedLock lock(mutex);
  maxThreadsCount=maxThreadsCount_;
}
//-----------------------------------------------------------------------------
size_t ResourceUsageExtractor::getMaxThreadsCount(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  return maxThreadsCount;
}
//-----------------------------------------------------------------------------
void ResourceUsageExtractor::setMaxProcessesCount(size_t maxProcessesCount_)
{
  Poco::Mutex::ScopedLock lock(mutex);
  maxProcessesCount=maxProcessesCount_;
}
//-----------------------------------------------------------------------------
size_t ResourceUsageExtractor::getMaxProcessesCount(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  return maxProcessesCount;
}
//-----------------------------------------------------------------------------
void ResourceUsageExtractor::setMaxDiskSize(size_t maxDiskSize_)
{
  Poco::Mutex::ScopedLock lock(mutex);
  diskSpaceExtractor.setDiskSize(maxDiskSize_);
}
//-----------------------------------------------------------------------------
size_t ResourceUsageExtractor::getMaxDiskSize(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  return diskSpaceExtractor.getDiskSize();
}
//-----------------------------------------------------------------------------
void ResourceUsageExtractor::setMaxVramSize(size_t maxVramSize_)
{
  Poco::Mutex::ScopedLock lock(mutex);
  memoryExtractor.setVramTotal(maxVramSize_);
}
//-----------------------------------------------------------------------------
size_t ResourceUsageExtractor::getMaxVramSize(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  return memoryExtractor.getVramTotal();
}
//-----------------------------------------------------------------------------
void ResourceUsageExtractor::setMaxRramSize(size_t maxRramSize_)
{
  Poco::Mutex::ScopedLock lock(mutex);
  memoryExtractor.setRramTotal(maxRramSize_);
}
//-----------------------------------------------------------------------------
size_t ResourceUsageExtractor::getMaxRramSize(void)
{
  Poco::Mutex::ScopedLock lock(mutex);
  return memoryExtractor.getRramTotal();
}
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
} // end namespace drce 
} // end namespace HCE