{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _rebuildPartitionIdMap(self, partitionIdList):\n    self._partitionIdMap = {}\n    for row, partitionId in enumerate(partitionIdList):\n      indices = self._partitionIdMap.get(partitionId, [])\n      indices.append(row)\n      self._partitionIdMap[partitionId] = indices", "response": "Rebuilds the partition Id map using the given list of partition ids."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the distances from inputPattern to all stored patterns.", "response": "def _calcDistance(self, inputPattern, distanceNorm=None):\n    \"\"\"Calculate the distances from inputPattern to all stored patterns. All\n    distances are between 0.0 and 1.0\n\n    :param inputPattern The pattern from which distances to all other patterns\n        are calculated\n\n    :param distanceNorm Degree of the distance norm\n    \"\"\"\n    if distanceNorm is None:\n      distanceNorm = self.distanceNorm\n\n    # Sparse memory\n    if self.useSparseMemory:\n      if self._protoSizes is None:\n        self._protoSizes = self._Memory.rowSums()\n      overlapsWithProtos = self._Memory.rightVecSumAtNZ(inputPattern)\n      inputPatternSum = inputPattern.sum()\n\n      if self.distanceMethod == \"rawOverlap\":\n        dist = inputPattern.sum() - overlapsWithProtos\n      elif self.distanceMethod == \"pctOverlapOfInput\":\n        dist = inputPatternSum - overlapsWithProtos\n        if inputPatternSum > 0:\n          dist /= inputPatternSum\n      elif self.distanceMethod == \"pctOverlapOfProto\":\n        overlapsWithProtos /= self._protoSizes\n        dist = 1.0 - overlapsWithProtos\n      elif self.distanceMethod == \"pctOverlapOfLarger\":\n        maxVal = numpy.maximum(self._protoSizes, inputPatternSum)\n        if maxVal.all() > 0:\n          overlapsWithProtos /= maxVal\n        dist = 1.0 - overlapsWithProtos\n      elif self.distanceMethod == \"norm\":\n        dist = self._Memory.vecLpDist(self.distanceNorm, inputPattern)\n        distMax = dist.max()\n        if distMax > 0:\n          dist /= distMax\n      else:\n        raise RuntimeError(\"Unimplemented distance method %s\" %\n          self.distanceMethod)\n\n    # Dense memory\n    else:\n      if self.distanceMethod == \"norm\":\n        dist = numpy.power(numpy.abs(self._M - inputPattern), self.distanceNorm)\n        dist = dist.sum(1)\n        dist = numpy.power(dist, 1.0/self.distanceNorm)\n        dist /= dist.max()\n      else:\n        raise RuntimeError (\"Not implemented yet for dense storage....\")\n\n    return dist"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncompute the distances from inputPattern to all stored patterns.", "response": "def _getDistances(self, inputPattern, partitionId=None):\n    \"\"\"Return the distances from inputPattern to all stored patterns.\n\n    :param inputPattern The pattern from which distances to all other patterns\n        are returned\n\n    :param partitionId If provided, ignore all training vectors with this\n        partitionId.\n    \"\"\"\n    if not self._finishedLearning:\n      self.finishLearning()\n      self._finishedLearning = True\n\n    if self._vt is not None and len(self._vt) > 0:\n      inputPattern = numpy.dot(self._vt, inputPattern - self._mean)\n\n    sparseInput = self._sparsifyVector(inputPattern)\n\n    # Compute distances\n    dist = self._calcDistance(sparseInput)\n    # Invalidate results where category is -1\n    if self._specificIndexTraining:\n      dist[numpy.array(self._categoryList) == -1] = numpy.inf\n\n    # Ignore vectors with this partition id by setting their distances to inf\n    if partitionId is not None:\n      dist[self._partitionIdMap.get(partitionId, [])] = numpy.inf\n\n    return dist"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef computeSVD(self, numSVDSamples=0, finalize=True):\n    if numSVDSamples == 0:\n      numSVDSamples = self._numPatterns\n\n    if not self.useSparseMemory:\n      self._a = self._Memory[:self._numPatterns]\n    else:\n      self._a = self._Memory.toDense()[:self._numPatterns]\n\n    self._mean = numpy.mean(self._a, axis=0)\n    self._a -= self._mean\n    u,self._s,self._vt = numpy.linalg.svd(self._a[:numSVDSamples])\n\n    if finalize:\n      self._finalizeSVD()\n\n    return self._s", "response": "Compute the singular value decomposition of the input patterns."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getAdaptiveSVDDims(self, singularValues, fractionOfMax=0.001):\n    v = singularValues/singularValues[0]\n    idx = numpy.where(v<fractionOfMax)[0]\n    if len(idx):\n      print \"Number of PCA dimensions chosen: \", idx[0], \"out of \", len(v)\n      return idx[0]\n    else:\n      print \"Number of PCA dimensions chosen: \", len(v)-1, \"out of \", len(v)\n      return len(v)-1", "response": "Compute the number of eigenvectors to keep."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _finalizeSVD(self, numSVDDims=None):\n    if numSVDDims is not None:\n      self.numSVDDims = numSVDDims\n\n\n    if self.numSVDDims==\"adaptive\":\n      if self.fractionOfMax is not None:\n          self.numSVDDims = self.getAdaptiveSVDDims(self._s, self.fractionOfMax)\n      else:\n          self.numSVDDims = self.getAdaptiveSVDDims(self._s)\n\n\n    if self._vt.shape[0] < self.numSVDDims:\n      print \"******************************************************************\"\n      print (\"Warning: The requested number of PCA dimensions is more than \"\n             \"the number of pattern dimensions.\")\n      print \"Setting numSVDDims = \", self._vt.shape[0]\n      print \"******************************************************************\"\n      self.numSVDDims = self._vt.shape[0]\n\n    self._vt = self._vt[:self.numSVDDims]\n\n    # Added when svd is not able to decompose vectors - uses raw spare vectors\n    if len(self._vt) == 0:\n      return\n\n    self._Memory = numpy.zeros((self._numPatterns,self.numSVDDims))\n    self._M = self._Memory\n    self.useSparseMemory = False\n\n    for i in range(self._numPatterns):\n      self._Memory[i] = numpy.dot(self._vt, self._a[i])\n\n    self._a = None", "response": "Called by finalizeLearning. This will finalize the svd and compute the eigenvectors."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef remapCategories(self, mapping):\n    categoryArray = numpy.array(self._categoryList)\n    newCategoryArray = numpy.zeros(categoryArray.shape[0])\n    newCategoryArray.fill(-1)\n    for i in xrange(len(mapping)):\n      newCategoryArray[categoryArray==i] = mapping[i]\n    self._categoryList = list(newCategoryArray)", "response": "Change the category indices in the image sensor categoryInfo array."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nchange the category associated with this vector.", "response": "def setCategoryOfVectors(self, vectorIndices, categoryIndices):\n    \"\"\"Change the category associated with this vector(s).\n\n    Used by the Network Builder to move vectors between categories, to enable\n    categories, and to invalidate vectors by setting the category to -1.\n\n    :param vectorIndices: Single index or list of indices\n\n    :param categoryIndices: Single index or list of indices. Can also be a\n        single index when vectorIndices is a list, in which case the same\n        category will be used for all vectors\n    \"\"\"\n    if not hasattr(vectorIndices, \"__iter__\"):\n      vectorIndices = [vectorIndices]\n      categoryIndices = [categoryIndices]\n    elif not hasattr(categoryIndices, \"__iter__\"):\n      categoryIndices = [categoryIndices] * len(vectorIndices)\n\n    for i in xrange(len(vectorIndices)):\n      vectorIndex = vectorIndices[i]\n      categoryIndex = categoryIndices[i]\n\n      # Out-of-bounds is not an error, because the KNN may not have seen the\n      # vector yet\n      if vectorIndex < len(self._categoryList):\n        self._categoryList[vectorIndex] = categoryIndex"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the next record in the list of records to encode.", "response": "def getNextRecord(self):\n    \"\"\"\n    Get the next record to encode. Includes getting a record from the \n    `dataSource` and applying filters. If the filters request more data from the \n    `dataSource` continue to get data from the `dataSource` until all filters \n    are satisfied. This method is separate from :meth:`.RecordSensor.compute` so that we can \n    use a standalone :class:`.RecordSensor` to get filtered data.\n    \"\"\"\n\n    allFiltersHaveEnoughData = False\n    while not allFiltersHaveEnoughData:\n      # Get the data from the dataSource\n      data = self.dataSource.getNextRecordDict()\n\n      if not data:\n        raise StopIteration(\"Datasource has no more data\")\n\n      # temporary check\n      if \"_reset\" not in data:\n        data[\"_reset\"] = 0\n      if \"_sequenceId\" not in data:\n        data[\"_sequenceId\"] = 0\n      if \"_category\" not in data:\n        data[\"_category\"] = [None]\n\n      data, allFiltersHaveEnoughData = self.applyFilters(data)\n\n    self.lastRecord = data\n\n    return data"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\napply the pre - encoding filters to the data and returns the data and all filters have enough data.", "response": "def applyFilters(self, data):\n    \"\"\"\n    Apply pre-encoding filters. These filters may modify or add data. If a \n    filter needs another record (e.g. a delta filter) it will request another \n    record by returning False and the current record will be skipped (but will \n    still be given to all filters).\n\n    We have to be very careful about resets. A filter may add a reset,\n    but other filters should not see the added reset, each filter sees\n    the original reset value, and we keep track of whether any filter\n    adds a reset.\n\n    :param data: (dict) The data that will be processed by the filter.\n    :returns: (tuple) with the data processed by the filter and a boolean to\n              know whether or not the filter needs mode data.\n    \"\"\"\n\n    if self.verbosity > 0:\n      print \"RecordSensor got data: %s\" % data\n\n    allFiltersHaveEnoughData = True\n    if len(self.preEncodingFilters) > 0:\n      originalReset = data['_reset']\n      actualReset = originalReset\n      for f in self.preEncodingFilters:\n        # if filter needs more data, it returns False\n        filterHasEnoughData = f.process(data)\n        allFiltersHaveEnoughData = (allFiltersHaveEnoughData\n                                    and filterHasEnoughData)\n        actualReset = actualReset or data['_reset']\n        data['_reset'] = originalReset\n      data['_reset'] = actualReset\n\n    return data, allFiltersHaveEnoughData"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef populateCategoriesOut(self, categories, output):\n    if categories[0] is None:\n      # The record has no entry in category field.\n      output[:] = -1\n    else:\n      # Populate category output array by looping over the smaller of the\n      # output array (size specified by numCategories) and the record's number\n      # of categories.\n      for i, cat in enumerate(categories[:len(output)]):\n        output[i] = cat\n      output[len(categories):] = -1", "response": "Populate the output array with the category indices."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef compute(self, inputs, outputs):\n    if not self.topDownMode:\n      data = self.getNextRecord()\n\n      # The private keys in data are standard of RecordStreamIface objects. Any\n      # add'l keys are column headers from the data source.\n      reset = data[\"_reset\"]\n      sequenceId = data[\"_sequenceId\"]\n      categories = data[\"_category\"]\n\n      # Encode the processed records; populate outputs[\"dataOut\"] in place\n      self.encoder.encodeIntoArray(data, outputs[\"dataOut\"])\n\n      # If there is a field to predict, set bucketIdxOut and actValueOut.\n      # There is a special case where a predicted field might be a vector, as in\n      # the CoordinateEncoder. Since this encoder does not provide bucket\n      # indices for prediction, we will ignore it.\n      if self.predictedField is not None and self.predictedField != \"vector\":\n        allEncoders = list(self.encoder.encoders)\n        if self.disabledEncoder is not None:\n          allEncoders.extend(self.disabledEncoder.encoders)\n        encoders = [e for e in allEncoders\n                    if e[0] == self.predictedField]\n        if len(encoders) == 0:\n          raise ValueError(\"There is no encoder for set for the predicted \"\n                           \"field: %s\" % self.predictedField)\n        # TODO: Figure out why there are sometimes multiple encoders with the\n        # same name.\n        #elif len(encoders) > 1:\n        #  raise ValueError(\"There cant' be more than 1 encoder for the \"\n        #                   \"predicted field: %s\" % self.predictedField)\n        else:\n          encoder = encoders[0][1]\n\n        actualValue = data[self.predictedField]\n        outputs[\"bucketIdxOut\"][:] = encoder.getBucketIndices(actualValue)\n        if isinstance(actualValue, str):\n          outputs[\"actValueOut\"][:] = encoder.getBucketIndices(actualValue)\n        else:\n          outputs[\"actValueOut\"][:] = actualValue\n\n      # Write out the scalar values obtained from they data source.\n      outputs[\"sourceOut\"][:] = self.encoder.getScalars(data)\n      self._outputValues[\"sourceOut\"] = self.encoder.getEncodedValues(data)\n\n      # -----------------------------------------------------------------------\n      # Get the encoded bit arrays for each field\n      encoders = self.encoder.getEncoderList()\n      prevOffset = 0\n      sourceEncodings = []\n      bitData = outputs[\"dataOut\"]\n      for encoder in encoders:\n        nextOffset = prevOffset + encoder.getWidth()\n        sourceEncodings.append(bitData[prevOffset:nextOffset])\n        prevOffset = nextOffset\n      self._outputValues['sourceEncodings'] = sourceEncodings\n\n      # Execute post-encoding filters, if any\n      for filter in self.postEncodingFilters:\n        filter.process(encoder=self.encoder, data=outputs['dataOut'])\n\n      # Populate the output numpy arrays; must assign by index.\n      outputs['resetOut'][0] = reset\n      outputs['sequenceIdOut'][0] = sequenceId\n      self.populateCategoriesOut(categories, outputs['categoryOut'])\n\n      # ------------------------------------------------------------------------\n      # Verbose print?\n      if self.verbosity >= 1:\n        if self._iterNum == 0:\n          self.encoder.pprintHeader(prefix=\"sensor:\")\n        if reset:\n          print \"RESET - sequenceID:%d\" % sequenceId\n        if self.verbosity >= 2:\n          print\n\n      # If verbosity >=2, print the record fields\n      if self.verbosity >= 1:\n        self.encoder.pprint(outputs[\"dataOut\"], prefix=\"%7d:\" % (self._iterNum))\n        scalarValues = self.encoder.getScalars(data)\n        nz = outputs[\"dataOut\"].nonzero()[0]\n        print \"     nz: (%d)\" % (len(nz)), nz\n        print \"  encIn:\", self.encoder.scalarsToStr(scalarValues)\n      if self.verbosity >= 2:\n        # if hasattr(data, 'header'):\n        #  header = data.header()\n        # else:\n        #  header = '     '.join(self.dataSource.names)\n        # print \"        \", header\n        print \"   data:\", str(data)\n      if self.verbosity >= 3:\n        decoded = self.encoder.decode(outputs[\"dataOut\"])\n        print \"decoded:\", self.encoder.decodedToStr(decoded)\n\n      self._iterNum += 1\n\n    else:\n\n      # ========================================================================\n      # Spatial\n      # ========================================================================\n      # This is the top down compute in sensor\n\n      # We get the spatial pooler's topDownOut as spatialTopDownIn\n      spatialTopDownIn = inputs['spatialTopDownIn']\n      spatialTopDownOut = self.encoder.topDownCompute(spatialTopDownIn)\n\n      # -----------------------------------------------------------------------\n      # Split topDownOutput into seperate outputs\n      values = [elem.value for elem in spatialTopDownOut]\n      scalars = [elem.scalar for elem in spatialTopDownOut]\n      encodings = [elem.encoding for elem in spatialTopDownOut]\n      self._outputValues['spatialTopDownOut'] = values\n      outputs['spatialTopDownOut'][:] = numpy.array(scalars)\n      self._outputValues['spatialTopDownEncodings'] = encodings\n\n      # ========================================================================\n      # Temporal\n      # ========================================================================\n\n      ## TODO: Add temporal top-down loop\n      # We get the temporal memory's topDownOut passed through the spatial\n      # pooler as temporalTopDownIn\n      temporalTopDownIn = inputs['temporalTopDownIn']\n      temporalTopDownOut = self.encoder.topDownCompute(temporalTopDownIn)\n\n      # -----------------------------------------------------------------------\n      # Split topDownOutput into separate outputs\n\n      values = [elem.value for elem in temporalTopDownOut]\n      scalars = [elem.scalar for elem in temporalTopDownOut]\n      encodings = [elem.encoding for elem in temporalTopDownOut]\n      self._outputValues['temporalTopDownOut'] = values\n      outputs['temporalTopDownOut'][:] = numpy.array(scalars)\n      self._outputValues['temporalTopDownEncodings'] = encodings\n\n      assert len(spatialTopDownOut) == len(temporalTopDownOut), (\n        \"Error: spatialTopDownOut and temporalTopDownOut should be the same \"\n        \"size\")", "response": "Compute the next record from the dataSource and encode it."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _convertNonNumericData(self, spatialOutput, temporalOutput, output):\n    encoders = self.encoder.getEncoderList()\n    types = self.encoder.getDecoderOutputFieldTypes()\n    for i, (encoder, type) in enumerate(zip(encoders, types)):\n      spatialData = spatialOutput[i]\n      temporalData = temporalOutput[i]\n\n      if type != FieldMetaType.integer and type != FieldMetaType.float:\n        # TODO: Make sure that this doesn't modify any state\n        spatialData = encoder.getScalars(spatialData)[0]\n        temporalData = encoder.getScalars(temporalData)[0]\n\n      assert isinstance(spatialData, (float, int))\n      assert isinstance(temporalData, (float, int))\n      output['spatialTopDownOut'][i] = spatialData\n      output['temporalTopDownOut'][i] = temporalData", "response": "Convert all of the non - numeric fields from spatialOutput and temporalOutput\n    into their scalar equivalents and records them in the output dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getOutputElementCount(self, name):\n\n    if name == \"resetOut\":\n      print (\"WARNING: getOutputElementCount should not have been called with \"\n             \"resetOut\")\n      return 1\n\n    elif name == \"sequenceIdOut\":\n      print (\"WARNING: getOutputElementCount should not have been called with \"\n             \"sequenceIdOut\")\n      return 1\n\n    elif name == \"dataOut\":\n      if self.encoder is None:\n        raise Exception(\"NuPIC requested output element count for 'dataOut' \"\n                        \"on a RecordSensor node, but the encoder has not \"\n                        \"been set\")\n      return self.encoder.getWidth()\n\n    elif name == \"sourceOut\":\n      if self.encoder is None:\n        raise Exception(\"NuPIC requested output element count for 'sourceOut' \"\n                        \"on a RecordSensor node, \"\n                        \"but the encoder has not been set\")\n      return len(self.encoder.getDescription())\n\n    elif name == \"bucketIdxOut\":\n      return 1\n\n    elif name == \"actValueOut\":\n      return 1\n\n    elif name == \"categoryOut\":\n      return self.numCategories\n\n    elif name == 'spatialTopDownOut' or name == 'temporalTopDownOut':\n      if self.encoder is None:\n        raise Exception(\"NuPIC requested output element count for 'sourceOut' \"\n                        \"on a RecordSensor node, \"\n                        \"but the encoder has not been set\")\n      return len(self.encoder.getDescription())\n    else:\n      raise Exception(\"Unknown output %s\" % name)", "response": "Returns the number of output elements in a single Sequences."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the value of a Spec parameter.", "response": "def setParameter(self, parameterName, index, parameterValue):\n    \"\"\"\n      Set the value of a Spec parameter. Most parameters are handled\n      automatically by PyRegion's parameter set mechanism. The ones that need\n      special treatment are explicitly handled here.\n    \"\"\"\n    if parameterName == 'topDownMode':\n      self.topDownMode = parameterValue\n    elif parameterName == 'predictedField':\n      self.predictedField = parameterValue\n    else:\n      raise Exception('Unknown parameter: ' + parameterName)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwrites this region to the given proto.", "response": "def writeToProto(self, proto):\n    \"\"\"\n    Overrides :meth:`nupic.bindings.regions.PyRegion.PyRegion.writeToProto`.\n    \"\"\"\n    self.encoder.write(proto.encoder)\n    if self.disabledEncoder is not None:\n      self.disabledEncoder.write(proto.disabledEncoder)\n    proto.topDownMode = int(self.topDownMode)\n    proto.verbosity = self.verbosity\n    proto.numCategories = self.numCategories"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef readFromProto(cls, proto):\n    instance = cls()\n\n    instance.encoder = MultiEncoder.read(proto.encoder)\n    if proto.disabledEncoder is not None:\n      instance.disabledEncoder = MultiEncoder.read(proto.disabledEncoder)\n    instance.topDownMode = bool(proto.topDownMode)\n    instance.verbosity = proto.verbosity\n    instance.numCategories = proto.numCategories\n\n    return instance", "response": "Reads a\n    from a proto."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef computeAccuracy(model, size, top):\n  accuracy = []\n\n  # Load MSNBC web data file\n  filename = os.path.join(os.path.dirname(__file__), \"msnbc990928.zip\")\n  with zipfile.ZipFile(filename) as archive:\n    with archive.open(\"msnbc990928.seq\") as datafile:\n      # Skip header lines (first 7 lines)\n      for _ in xrange(7):\n        next(datafile)\n\n      # Skip learning data and compute accuracy using only new sessions\n      for _ in xrange(LEARNING_RECORDS):\n        next(datafile)\n\n      # Compute prediction accuracy by checking if the next page in the sequence\n      # is within the top N predictions calculated by the model\n      for _ in xrange(size):\n        pages = readUserSession(datafile)\n        model.resetSequenceStates()\n        for i in xrange(len(pages) - 1):\n          result = model.run({\"page\": pages[i]})\n          inferences = result.inferences[\"multiStepPredictions\"][1]\n\n          # Get top N predictions for the next page\n          predicted = sorted(inferences.items(), key=itemgetter(1), reverse=True)[:top]\n\n          # Check if the next page is within the predicted pages\n          accuracy.append(1 if pages[i + 1] in zip(*predicted)[0] else 0)\n\n  return np.mean(accuracy)", "response": "Compute prediction accuracy using the given model and size."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef readUserSession(datafile):\n  for line in datafile:\n    pages = line.split()\n    total = len(pages)\n    # Select user sessions with 2 or more pages\n    if total < 2:\n      continue\n\n    # Exclude outliers by removing extreme long sessions\n    if total > 500:\n      continue\n\n    return [PAGE_CATEGORIES[int(i) - 1] for i in pages]\n  return []", "response": "Reads the user session record from the file s cursor position."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef rewind(self):\n\n    # Superclass rewind\n    super(FileRecordStream, self).rewind()\n\n    self.close()\n    self._file = open(self._filename, self._mode)\n    self._reader = csv.reader(self._file, dialect=\"excel\")\n\n    # Skip header rows\n    self._reader.next()\n    self._reader.next()\n    self._reader.next()\n\n    # Reset record count, etc.\n    self._recordCount = 0", "response": "Rewinds the file to the beginning of the file."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the next available data record from the file.", "response": "def getNextRecord(self, useCache=True):\n    \"\"\" Returns next available data record from the file.\n\n    :returns: a data row (a list or tuple) if available; None, if no more\n              records in the table (End of Stream - EOS); empty sequence (list\n              or tuple) when timing out while waiting for the next record.\n    \"\"\"\n    assert self._file is not None\n    assert self._mode == self._FILE_READ_MODE\n\n    # Read the line\n    try:\n      line = self._reader.next()\n\n    except StopIteration:\n      if self.rewindAtEOF:\n        if self._recordCount == 0:\n          raise Exception(\"The source configured to reset at EOF but \"\n                          \"'%s' appears to be empty\" % self._filename)\n        self.rewind()\n        line = self._reader.next()\n\n      else:\n        return None\n\n    # Keep score of how many records were read\n    self._recordCount += 1\n\n    # Split the line to text fields and convert each text field to a Python\n    # object if value is missing (empty string) encode appropriately for\n    # upstream consumers in the case of numeric types, this means replacing\n    # missing data with a sentinel value for string type, we can leave the empty\n    # string in place\n    record = []\n    for i, f in enumerate(line):\n      #print \"DEBUG: Evaluating field @ index %s: %r\" % (i, f)\n      #sys.stdout.flush()\n      if f in self._missingValues:\n        record.append(SENTINEL_VALUE_FOR_MISSING_DATA)\n      else:\n        # either there is valid data, or the field is string type,\n        # in which case the adapter does the right thing by default\n        record.append(self._adapters[i](f))\n\n    return record"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef appendRecord(self, record):\n\n    assert self._file is not None\n    assert self._mode == self._FILE_WRITE_MODE\n    assert isinstance(record, (list, tuple)), \\\n      \"unexpected record type: \" + repr(type(record))\n\n    assert len(record) == self._fieldCount, \\\n      \"len(record): %s, fieldCount: %s\" % (len(record), self._fieldCount)\n\n    # Write header if needed\n    if self._recordCount == 0:\n      # Write the header\n      names, types, specials = zip(*self.getFields())\n      for line in names, types, specials:\n        self._writer.writerow(line)\n\n    # Keep track of sequences, make sure time flows forward\n    self._updateSequenceInfo(record)\n\n    line = [self._adapters[i](f) for i, f in enumerate(record)]\n\n    self._writer.writerow(line)\n    self._recordCount += 1", "response": "Adds a record to the underlying CSV file."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef appendRecords(self, records, progressCB=None):\n\n    for record in records:\n      self.appendRecord(record)\n      if progressCB is not None:\n        progressCB()", "response": "Adds multiple records to the internal storage."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getBookmark(self):\n\n    if self._write and self._recordCount==0:\n      return None\n\n    rowDict = dict(filepath=os.path.realpath(self._filename),\n                   currentRow=self._recordCount)\n    return json.dumps(rowDict)", "response": "Gets a bookmark or anchor to the current position."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef seekFromEnd(self, numRecords):\n    self._file.seek(self._getTotalLineCount() - numRecords)\n    return self.getBookmark()", "response": "Seeks to the end of the file and returns a bookmark to the desired location."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads the file and returns a dictionary of stats.", "response": "def getStats(self):\n    \"\"\"\n    Parse the file using dedicated reader and collect fields stats. Never\n    called if user of :class:`~.FileRecordStream` does not invoke\n    :meth:`~.FileRecordStream.getStats` method.\n\n    :returns:\n          a dictionary of stats. In the current implementation, min and max\n          fields are supported. Example of the return dictionary is:\n\n          .. code-block:: python\n\n             {\n               'min' : [f1_min, f2_min, None, None, fn_min],\n               'max' : [f1_max, f2_max, None, None, fn_max]\n             }\n\n          (where fx_min/fx_max are set for scalar fields, or None if not)\n\n    \"\"\"\n\n    # Collect stats only once per File object, use fresh csv iterator\n    # to keep the next() method returning sequential records no matter when\n    # caller asks for stats\n    if self._stats == None:\n      # Stats are only available when reading csv file\n      assert self._mode == self._FILE_READ_MODE\n\n      inFile = open(self._filename, self._FILE_READ_MODE)\n\n      # Create a new reader; read names, types, specials\n      reader = csv.reader(inFile, dialect=\"excel\")\n      names = [n.strip() for n in reader.next()]\n      types = [t.strip() for t in reader.next()]\n      # Skip over specials\n      reader.next()\n\n      # Initialize stats to all None\n      self._stats = dict()\n      self._stats['min'] = []\n      self._stats['max'] = []\n\n      for i in xrange(len(names)):\n        self._stats['min'].append(None)\n        self._stats['max'].append(None)\n\n      # Read the file, collect stats\n      while True:\n        try:\n          line = reader.next()\n          for i, f in enumerate(line):\n            if (len(types) > i and\n                types[i] in [FieldMetaType.integer, FieldMetaType.float] and\n                f not in self._missingValues):\n              value = self._adapters[i](f)\n              if self._stats['max'][i] == None or \\\n                 self._stats['max'][i] < value:\n                self._stats['max'][i] = value\n              if self._stats['min'][i] == None or \\\n                 self._stats['min'][i] > value:\n                self._stats['min'][i] = value\n\n        except StopIteration:\n          break\n\n    return self._stats"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _updateSequenceInfo(self, r):\n\n    # Get current sequence id (if any)\n    newSequence = False\n    sequenceId = (r[self._sequenceIdIdx]\n                  if self._sequenceIdIdx is not None else None)\n    if sequenceId != self._currSequence:\n      # verify that the new sequence didn't show up before\n      if sequenceId in self._sequences:\n        raise Exception('Broken sequence: %s, record: %s' % \\\n                        (sequenceId, r))\n\n      # add the finished sequence to the set of sequence\n      self._sequences.add(self._currSequence)\n      self._currSequence = sequenceId\n\n      # Verify that the reset is consistent (if there is one)\n      if self._resetIdx:\n        assert r[self._resetIdx] == 1\n      newSequence = True\n\n    else:\n      # Check the reset\n      reset = False\n      if self._resetIdx:\n        reset = r[self._resetIdx]\n        if reset == 1:\n          newSequence = True\n\n    # If it's still the same old sequence make sure the time flows forward\n    if not newSequence:\n      if self._timeStampIdx and self._currTime is not None:\n        t = r[self._timeStampIdx]\n        if t < self._currTime:\n          raise Exception('No time travel. Early timestamp for record: %s' % r)\n\n    if self._timeStampIdx:\n      self._currTime = r[self._timeStampIdx]", "response": "Update the internal state of the sequence info in the current record."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nextracting start row from the bookmark information", "response": "def _getStartRow(self, bookmark):\n    \"\"\" Extracts start row from the bookmark information\n    \"\"\"\n    bookMarkDict = json.loads(bookmark)\n\n    realpath = os.path.realpath(self._filename)\n\n    bookMarkFile = bookMarkDict.get('filepath', None)\n\n    if bookMarkFile != realpath:\n      print (\"Ignoring bookmark due to mismatch between File's \"\n             \"filename realpath vs. bookmark; realpath: %r; bookmark: %r\") % (\n        realpath, bookMarkDict)\n      return 0\n    else:\n      return bookMarkDict['currentRow']"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _getTotalLineCount(self):\n    # Flush the file before we open it again to count lines\n    if self._mode == self._FILE_WRITE_MODE:\n      self._file.flush()\n    return sum(1 for line in open(self._filename, self._FILE_READ_MODE))", "response": "Returns the number of lines in dataset including header lines"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getDataRowCount(self):\n    numLines = self._getTotalLineCount()\n\n    if numLines == 0:\n      # this may be the case in a file opened for write before the\n      # header rows are written out\n      assert self._mode == self._FILE_WRITE_MODE and self._recordCount == 0\n      numDataRows = 0\n    else:\n      numDataRows = numLines - self._NUM_HEADER_ROWS\n\n    assert numDataRows >= 0\n\n    return numDataRows", "response": "returns the number of data rows in the current dataset"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nhandling looping over the input data and passing each row into the given model object, as well as extracting the result object and passing it into an output handler. :param inputData: file path to input data CSV :param model: OPF Model object :param gymName: Gym name, used for output handler naming :param plot: Whether to use matplotlib or not. If false, uses file output.", "response": "def runIoThroughNupic(inputData, model, gymName, plot):\n  \"\"\"\n  Handles looping over the input data and passing each row into the given model\n  object, as well as extracting the result object and passing it into an output\n  handler.\n  :param inputData: file path to input data CSV\n  :param model: OPF Model object\n  :param gymName: Gym name, used for output handler naming\n  :param plot: Whether to use matplotlib or not. If false, uses file output.\n  \"\"\"\n  inputFile = open(inputData, \"rb\")\n  csvReader = csv.reader(inputFile)\n  # skip header rows\n  csvReader.next()\n  csvReader.next()\n  csvReader.next()\n\n  shifter = InferenceShifter()\n  if plot:\n    output = nupic_anomaly_output.NuPICPlotOutput(gymName)\n  else:\n    output = nupic_anomaly_output.NuPICFileOutput(gymName)\n\n  counter = 0\n  for row in csvReader:\n    counter += 1\n    if (counter % 100 == 0):\n      print \"Read %i lines...\" % counter\n    timestamp = datetime.datetime.strptime(row[0], DATE_FORMAT)\n    consumption = float(row[1])\n    result = model.run({\n      \"timestamp\": timestamp,\n      \"kw_energy_consumption\": consumption\n    })\n\n    if plot:\n      result = shifter.shift(result)\n\n    prediction = result.inferences[\"multiStepBestPredictions\"][1]\n    anomalyScore = result.inferences[\"anomalyScore\"]\n    output.write(timestamp, consumption, prediction, anomalyScore)\n\n  inputFile.close()\n  output.close()"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef topDownCompute(self, encoded):\n\n    #Decode to delta scalar\n    if self._prevAbsolute==None or self._prevDelta==None:\n      return [EncoderResult(value=0, scalar=0,\n                             encoding=numpy.zeros(self.n))]\n    ret = self._adaptiveScalarEnc.topDownCompute(encoded)\n    if self._prevAbsolute != None:\n      ret = [EncoderResult(value=ret[0].value+self._prevAbsolute,\n                          scalar=ret[0].scalar+self._prevAbsolute,\n                          encoding=ret[0].encoding)]\n#      ret[0].value+=self._prevAbsolute\n#      ret[0].scalar+=self._prevAbsolute\n    return ret", "response": "This method computes the top down of the scalar encoder."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef isTemporal(inferenceElement):\n    if InferenceElement.__temporalInferenceElements is None:\n      InferenceElement.__temporalInferenceElements = \\\n                                set([InferenceElement.prediction])\n\n    return inferenceElement in InferenceElement.__temporalInferenceElements", "response": "Returns True if the inference element is predicted the input\n    for the NEXT timestep."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the number of records that elapse between when an inference is made and when the corresponding input record will appear.", "response": "def getTemporalDelay(inferenceElement, key=None):\n    \"\"\" Returns the number of records that elapse between when an inference is\n    made and when the corresponding input record will appear. For example, a\n    multistep prediction for 3 timesteps out will have a delay of 3\n\n\n    Parameters:\n    -----------------------------------------------------------------------\n\n    inferenceElement:   The InferenceElement value being delayed\n    key:                If the inference is a dictionary type, this specifies\n                        key for the sub-inference that is being delayed\n    \"\"\"\n    # -----------------------------------------------------------------------\n    # For next step prediction, we shift by 1\n    if inferenceElement in (InferenceElement.prediction,\n                            InferenceElement.encodings):\n      return 1\n    # -----------------------------------------------------------------------\n    # For classification, anomaly scores, the inferences immediately succeed the\n    # inputs\n    if inferenceElement in (InferenceElement.anomalyScore,\n                            InferenceElement.anomalyLabel,\n                            InferenceElement.classification,\n                            InferenceElement.classConfidences):\n      return 0\n    # -----------------------------------------------------------------------\n    # For multistep prediction, the delay is based on the key in the inference\n    # dictionary\n    if inferenceElement in (InferenceElement.multiStepPredictions,\n                            InferenceElement.multiStepBestPredictions):\n      return int(key)\n\n    # -----------------------------------------------------------------------\n    # default: return 0\n    return 0"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getMaxDelay(inferences):\n    maxDelay = 0\n    for inferenceElement, inference in inferences.iteritems():\n      if isinstance(inference, dict):\n        for key in inference.iterkeys():\n          maxDelay = max(InferenceElement.getTemporalDelay(inferenceElement,\n                                                            key),\n                         maxDelay)\n      else:\n        maxDelay = max(InferenceElement.getTemporalDelay(inferenceElement),\n                       maxDelay)\n\n\n    return maxDelay", "response": "Returns the maximum delay for the InferenceElements in the inferences dictionary"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef isTemporal(inferenceType):\n    if InferenceType.__temporalInferenceTypes is None:\n      InferenceType.__temporalInferenceTypes = \\\n                                set([InferenceType.TemporalNextStep,\n                                     InferenceType.TemporalClassification,\n                                     InferenceType.TemporalAnomaly,\n                                     InferenceType.TemporalMultiStep,\n                                     InferenceType.NontemporalMultiStep])\n\n    return inferenceType in InferenceType.__temporalInferenceTypes", "response": "Returns True if the inference type is temporal."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nmake a directory from the given absolute path.", "response": "def makeDirectoryFromAbsolutePath(absDirPath):\n  \"\"\" Makes directory for the given directory path with default permissions.\n  If the directory already exists, it is treated as success.\n\n  absDirPath:   absolute path of the directory to create.\n\n  Returns:      absDirPath arg\n\n  Exceptions:         OSError if directory creation fails\n  \"\"\"\n\n  assert os.path.isabs(absDirPath)\n\n  try:\n    os.makedirs(absDirPath)\n  except OSError, e:\n    if e.errno != os.errno.EEXIST:\n      raise\n\n  return absDirPath"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread in the given configuration file and return a dict describing the file.", "response": "def _readConfigFile(cls, filename, path=None):\n    \"\"\" Parse the given XML file and return a dict describing the file.\n\n    Parameters:\n    ----------------------------------------------------------------\n    filename:  name of XML file to parse (no path)\n    path:      path of the XML file. If None, then use the standard\n                  configuration search path.\n    retval:    returns a dict with each property as a key and a dict of all\n               the property's attributes as value\n    \"\"\"\n\n    outputProperties = dict()\n\n    # Get the path to the config files.\n    if path is None:\n      filePath = cls.findConfigFile(filename)\n    else:\n      filePath = os.path.join(path, filename)\n\n    # ------------------------------------------------------------------\n    # Read in the config file\n    try:\n      if filePath is not None:\n        try:\n          # Use warn since console log level is set to warning\n          _getLoggerBase().debug(\"Loading config file: %s\", filePath)\n          with open(filePath, 'r') as inp:\n            contents = inp.read()\n        except Exception:\n          raise RuntimeError(\"Expected configuration file at %s\" % filePath)\n      else:\n        # If the file was not found in the normal search paths, which includes\n        # checking the NTA_CONF_PATH, we'll try loading it from pkg_resources.\n        try:\n          contents = resource_string(\"nupic.support\", filename)\n        except Exception as resourceException:\n          # We expect these to be read, and if they don't exist we'll just use\n          # an empty configuration string.\n          if filename in [USER_CONFIG, CUSTOM_CONFIG]:\n            contents = '<configuration/>'\n          else:\n            raise resourceException\n\n      elements = ElementTree.XML(contents)\n\n      if elements.tag != 'configuration':\n        raise RuntimeError(\"Expected top-level element to be 'configuration' \"\n                           \"but got '%s'\" % (elements.tag))\n\n      # ------------------------------------------------------------------\n      # Add in each property found\n      propertyElements = elements.findall('./property')\n\n      for propertyItem in propertyElements:\n\n        propInfo = dict()\n\n        # Parse this property element\n        propertyAttributes = list(propertyItem)\n        for propertyAttribute in propertyAttributes:\n          propInfo[propertyAttribute.tag] = propertyAttribute.text\n\n        # Get the name\n        name = propInfo.get('name', None)\n\n        # value is allowed to be empty string\n        if 'value' in propInfo and propInfo['value'] is None:\n          value = ''\n        else:\n          value = propInfo.get('value', None)\n\n          if value is None:\n            if 'novalue' in propInfo:\n              # Placeholder \"novalue\" properties are intended to be overridden\n              # via dynamic configuration or another configuration layer.\n              continue\n            else:\n              raise RuntimeError(\"Missing 'value' element within the property \"\n                                 \"element: => %s \" % (str(propInfo)))\n\n        # The value is allowed to contain substitution tags of the form\n        # ${env.VARNAME}, which should be substituted with the corresponding\n        # environment variable values\n        restOfValue = value\n        value = ''\n        while True:\n          # Find the beginning of substitution tag\n          pos = restOfValue.find('${env.')\n          if pos == -1:\n            # No more environment variable substitutions\n            value += restOfValue\n            break\n\n          # Append prefix to value accumulator\n          value += restOfValue[0:pos]\n\n          # Find the end of current substitution tag\n          varTailPos = restOfValue.find('}', pos)\n          if varTailPos == -1:\n            raise RuntimeError(\n              \"Trailing environment variable tag delimiter '}'\"\n              \" not found in %r\" % (restOfValue))\n\n          # Extract environment variable name from tag\n          varname = restOfValue[pos + 6:varTailPos]\n          if varname not in os.environ:\n            raise RuntimeError(\"Attempting to use the value of the environment\"\n                               \" variable %r, which is not defined\" % (\n                               varname))\n          envVarValue = os.environ[varname]\n\n          value += envVarValue\n\n          restOfValue = restOfValue[varTailPos + 1:]\n\n        # Check for errors\n        if name is None:\n          raise RuntimeError(\n            \"Missing 'name' element within following property \"\n            \"element:\\n => %s \" % (str(propInfo)))\n\n        propInfo['value'] = value\n        outputProperties[name] = propInfo\n\n      return outputProperties\n    except Exception:\n      _getLoggerBase().exception(\"Error while parsing configuration file: %s.\",\n                             filePath)\n      raise"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting multiple custom properties and persist them to the custom configuration store.", "response": "def setCustomProperties(cls, properties):\n    \"\"\" Set multiple custom properties and persist them to the custom\n    configuration store.\n\n    Parameters:\n    ----------------------------------------------------------------\n    properties: a dict of property name/value pairs to set\n    \"\"\"\n    _getLogger().info(\"Setting custom configuration properties=%r; caller=%r\",\n                      properties, traceback.format_stack())\n\n    _CustomConfigurationFileWrapper.edit(properties)\n\n    for propertyName, value in properties.iteritems():\n      cls.set(propertyName, value)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nclear all properties from in - memory cache but do NOT alter the custom configuration file.", "response": "def clear(cls):\n    \"\"\" Clear all configuration properties from in-memory cache, but do NOT\n    alter the custom configuration file. Used in unit-testing.\n    \"\"\"\n    # Clear the in-memory settings cache, forcing reload upon subsequent \"get\"\n    # request.\n    super(Configuration, cls).clear()\n\n    # Reset in-memory custom configuration info.\n    _CustomConfigurationFileWrapper.clear(persistent=False)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef resetCustomConfig(cls):\n    _getLogger().info(\"Resetting all custom configuration properties; \"\n                      \"caller=%r\", traceback.format_stack())\n\n    # Clear the in-memory settings cache, forcing reload upon subsequent \"get\"\n    # request.\n    super(Configuration, cls).clear()\n\n    # Delete the persistent custom configuration store and reset in-memory\n    # custom configuration info\n    _CustomConfigurationFileWrapper.clear(persistent=True)", "response": "Clear all custom configuration settings and delete the persistent custom configuration store."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef clear(cls, persistent=False):\n    if persistent:\n      try:\n        os.unlink(cls.getPath())\n      except OSError, e:\n        if e.errno != errno.ENOENT:\n          _getLogger().exception(\"Error %s while trying to remove dynamic \" \\\n                                 \"configuration file: %s\", e.errno,\n                                 cls.getPath())\n          raise\n    cls._path = None", "response": "Delete the temporary file containing the dynamic configuration file."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getCustomDict(cls):\n    if not os.path.exists(cls.getPath()):\n      return dict()\n\n    properties = Configuration._readConfigFile(os.path.basename(\n      cls.getPath()), os.path.dirname(cls.getPath()))\n\n    values = dict()\n    for propName in properties:\n      if 'value' in properties[propName]:\n        values[propName] = properties[propName]['value']\n\n    return values", "response": "Returns a dict of all temporary values in custom configuration file"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nedit the XML configuration file with the parameters specified by properties.", "response": "def edit(cls, properties):\n    \"\"\" Edits the XML configuration file with the parameters specified by\n    properties\n\n    Parameters:\n    ----------------------------------------------------------------\n    properties: dict of settings to be applied to the custom configuration store\n                 (key is property name, value is value)\n    \"\"\"\n    copyOfProperties = copy(properties)\n\n    configFilePath = cls.getPath()\n\n    try:\n      with open(configFilePath, 'r') as fp:\n        contents = fp.read()\n    except IOError, e:\n      if e.errno != errno.ENOENT:\n        _getLogger().exception(\"Error %s reading custom configuration store \"\n                               \"from %s, while editing properties %s.\",\n                               e.errno, configFilePath, properties)\n        raise\n      contents = '<configuration/>'\n\n    try:\n      elements = ElementTree.XML(contents)\n      ElementTree.tostring(elements)\n    except Exception, e:\n      # Raising error as RuntimeError with custom message since ElementTree\n      # exceptions aren't clear.\n      msg = \"File contents of custom configuration is corrupt.  File \" \\\n            \"location: %s; Contents: '%s'. Original Error (%s): %s.\" % \\\n            (configFilePath, contents, type(e), e)\n      _getLogger().exception(msg)\n      raise RuntimeError(msg), None, sys.exc_info()[2]\n\n    if elements.tag != 'configuration':\n      e = \"Expected top-level element to be 'configuration' but got '%s'\" % \\\n          (elements.tag)\n      _getLogger().error(e)\n      raise RuntimeError(e)\n\n    # Apply new properties to matching settings in the custom config store;\n    # pop matching properties from our copy of the properties dict\n    for propertyItem in elements.findall('./property'):\n      propInfo = dict((attr.tag, attr.text) for attr in propertyItem)\n      name = propInfo['name']\n      if name in copyOfProperties:\n        foundValues = propertyItem.findall('./value')\n        if len(foundValues) > 0:\n          foundValues[0].text = str(copyOfProperties.pop(name))\n          if not copyOfProperties:\n            break\n        else:\n          e = \"Property %s missing value tag.\" % (name,)\n          _getLogger().error(e)\n          raise RuntimeError(e)\n\n    # Add unmatched remaining properties to custom config store\n    for propertyName, value in copyOfProperties.iteritems():\n      newProp = ElementTree.Element('property')\n      nameTag = ElementTree.Element('name')\n      nameTag.text = propertyName\n      newProp.append(nameTag)\n\n      valueTag = ElementTree.Element('value')\n      valueTag.text = str(value)\n      newProp.append(valueTag)\n\n      elements.append(newProp)\n\n    try:\n      makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))\n      with open(configFilePath, 'w') as fp:\n        fp.write(ElementTree.tostring(elements))\n    except Exception, e:\n      _getLogger().exception(\"Error while saving custom configuration \"\n                             \"properties %s in %s.\", properties,\n                             configFilePath)\n      raise"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _setPath(cls):\n    cls._path = os.path.join(os.environ['NTA_DYNAMIC_CONF_DIR'],\n                             cls.customFileName)", "response": "Sets the path of the custom configuration file in the current language."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the particle state as a dict. This is enough information to acquire this particle on another worker.", "response": "def getState(self):\n    \"\"\"Get the particle state as a dict. This is enough information to\n    instantiate this particle on another worker.\"\"\"\n    varStates = dict()\n    for varName, var in self.permuteVars.iteritems():\n      varStates[varName] = var.getState()\n\n    return dict(id=self.particleId,\n                genIdx=self.genIdx,\n                swarmId=self.swarmId,\n                varStates=varStates)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef initStateFrom(self, particleId, particleState, newBest):\n    # Get the update best position and result?\n    if newBest:\n      (bestResult, bestPosition) = self._resultsDB.getParticleBest(particleId)\n    else:\n      bestResult = bestPosition = None\n\n    # Replace with the position and velocity of each variable from\n    #  saved state\n    varStates = particleState['varStates']\n    for varName in varStates.keys():\n      varState = copy.deepcopy(varStates[varName])\n      if newBest:\n        varState['bestResult'] = bestResult\n      if bestPosition is not None:\n        varState['bestPosition'] = bestPosition[varName]\n      self.permuteVars[varName].setState(varState)", "response": "Initialize all of our variables positions velocities and bestResult and bestPosition from the given particle state."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncopy specific variables from particleState into this particle.", "response": "def copyVarStatesFrom(self, particleState, varNames):\n    \"\"\"Copy specific variables from particleState into this particle.\n\n    Parameters:\n    --------------------------------------------------------------\n    particleState:        dict produced by a particle's getState() method\n    varNames:             which variables to copy\n    \"\"\"\n    # Set this to false if you don't want the variable to move anymore\n    #  after we set the state\n    allowedToMove = True\n\n    for varName in particleState['varStates']:\n      if varName in varNames:\n\n        # If this particle doesn't include this field, don't copy it\n        if varName not in self.permuteVars:\n          continue\n\n        # Set the best position to the copied position\n        state = copy.deepcopy(particleState['varStates'][varName])\n        state['_position'] = state['position']\n        state['bestPosition'] = state['position']\n\n        if not allowedToMove:\n          state['velocity'] = 0\n\n        # Set the state now\n        self.permuteVars[varName].setState(state)\n\n        if allowedToMove:\n          # Let the particle move in both directions from the best position\n          #  it found previously and set it's initial velocity to a known\n          #  fraction of the total distance.\n          self.permuteVars[varName].resetVelocity(self._rng)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the position of this particle. This returns a dictionary of key - value pairs where each key is the name of the flattened permutation variable and each value is the value of the flattened permutation variable.", "response": "def getPosition(self):\n    \"\"\"Return the position of this particle. This returns a dict() of key\n    value pairs where each key is the name of the flattened permutation\n    variable and the value is its chosen value.\n\n    Parameters:\n    --------------------------------------------------------------\n    retval:     dict() of flattened permutation choices\n    \"\"\"\n    result = dict()\n    for (varName, value) in self.permuteVars.iteritems():\n      result[varName] = value.getPosition()\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the position of a particle given its state dict.", "response": "def getPositionFromState(pState):\n    \"\"\"Return the position of a particle given its state dict.\n\n    Parameters:\n    --------------------------------------------------------------\n    retval:     dict() of particle position, keys are the variable names,\n                  values are their positions\n    \"\"\"\n    result = dict()\n    for (varName, value) in pState['varStates'].iteritems():\n      result[varName] = value['position']\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nagitates this particle so that it is likely to go to a new position.", "response": "def agitate(self):\n    \"\"\"Agitate this particle so that it is likely to go to a new position.\n    Every time agitate is called, the particle is jiggled an even greater\n    amount.\n\n    Parameters:\n    --------------------------------------------------------------\n    retval:               None\n    \"\"\"\n    for (varName, var) in self.permuteVars.iteritems():\n      var.agitate()\n\n    self.newPosition()"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nchoose a new position based on the results obtained so far from all other variables.", "response": "def newPosition(self, whichVars=None):\n    # TODO: incorporate data from choice variables....\n    # TODO: make sure we're calling this when appropriate.\n    \"\"\"Choose a new position based on results obtained so far from all other\n    particles.\n\n    Parameters:\n    --------------------------------------------------------------\n    whichVars:       If not None, only move these variables\n    retval:               new position\n    \"\"\"\n    # Get the global best position for this swarm generation\n    globalBestPosition = None\n    # If speculative particles are enabled, use the global best considering\n    #  even particles in the current generation. This gives better results\n    #  but does not provide repeatable results because it depends on\n    #  worker timing\n    if self._hsObj._speculativeParticles:\n      genIdx = self.genIdx\n    else:\n      genIdx = self.genIdx - 1\n\n    if genIdx >= 0:\n      (bestModelId, _) = self._resultsDB.bestModelIdAndErrScore(self.swarmId,\n                                                                genIdx)\n      if bestModelId is not None:\n        (particleState, _, _, _, _) = self._resultsDB.getParticleInfo(\n          bestModelId)\n        globalBestPosition = Particle.getPositionFromState(particleState)\n\n    # Update each variable\n    for (varName, var) in self.permuteVars.iteritems():\n      if whichVars is not None and varName not in whichVars:\n        continue\n      if globalBestPosition is None:\n        var.newPosition(None, self._rng)\n      else:\n        var.newPosition(globalBestPosition[varName], self._rng)\n\n    # get the new position\n    position = self.getPosition()\n\n    # Log the new position\n    if self.logger.getEffectiveLevel() <= logging.DEBUG:\n      msg = StringIO.StringIO()\n      print >> msg, \"New particle position: \\n%s\" % (pprint.pformat(position,\n                                                                    indent=4))\n      print >> msg, \"Particle variables:\"\n      for (varName, var) in self.permuteVars.iteritems():\n        print >> msg, \"  %s: %s\" % (varName, str(var))\n      self.logger.debug(msg.getvalue())\n      msg.close()\n\n    return position"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef propose(self, current, r):\n    stay = (r.uniform(0, 1) < self.kernel)\n    if stay:\n      logKernel = numpy.log(self.kernel)\n      return current, logKernel, logKernel\n    else: # Choose uniformly, not according to the pmf.\n      curIndex = self.keyMap[current]\n      ri = r.randint(0, self.nKeys-1)\n      logKernel = numpy.log(1.0 - self.kernel)\n      lp = logKernel + self.logp\n      if ri < curIndex: return self.keys[ri], lp, lp\n      else: return self.keys[ri+1], lp, lp", "response": "Generates a random sample from the discrete probability distribution and\n    returns its value the log of the probability of sampling that value and the log of the probability of sampling that value and the log of the probability of sampling that value."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ngenerate a random sample from the Poisson probability distribution withlocation and scale equal to the current value.", "response": "def propose(self, current, r):\n    \"\"\"Generates a random sample from the Poisson probability distribution with\n    with location and scale parameter equal to the current value (passed in).\n    Returns the value of the random sample, the log of the probability of\n    sampling that value, and the log of the probability of sampling the current\n    value if the roles of the new sample and the current sample were reversed\n    (the log of the backward proposal probability).\n    \"\"\"\n    curLambda = current + self.offset\n    x, logProb = PoissonDistribution(curLambda).sample(r)\n    logBackward = PoissonDistribution(x+self.offset).logDensity(current)\n    return x, logProb, logBackward"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef __getLogger(cls):\n    if cls.__logger is None:\n      cls.__logger = opf_utils.initLogger(cls)\n    return cls.__logger", "response": "Returns the logger for this object."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a new model instance given a dictionary describing the current model and the log level.", "response": "def create(modelConfig, logLevel=logging.ERROR):\n    \"\"\" Create a new model instance, given a description dictionary.\n\n    :param modelConfig: (dict)\n           A dictionary describing the current model,\n           `described here <../../quick-start/example-model-params.html>`_.\n\n    :param logLevel: (int) The level of logging output that should be generated\n\n    :raises Exception: Unsupported model type\n\n    :returns: :class:`nupic.frameworks.opf.model.Model`\n    \"\"\"\n    logger = ModelFactory.__getLogger()\n    logger.setLevel(logLevel)\n    logger.debug(\"ModelFactory returning Model from dict: %s\", modelConfig)\n\n    modelClass = None\n    if modelConfig['model'] == \"HTMPrediction\":\n      modelClass = HTMPredictionModel\n    elif modelConfig['model'] == \"TwoGram\":\n      modelClass = TwoGramModel\n    elif modelConfig['model'] == \"PreviousValue\":\n      modelClass = PreviousValueModel\n    else:\n      raise Exception(\"ModelFactory received unsupported Model type: %s\" % \\\n                      modelConfig['model'])\n\n    return modelClass(**modelConfig['modelParams'])"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nloading the saved model from the directory specified by the checkpoint.", "response": "def loadFromCheckpoint(savedModelDir, newSerialization=False):\n    \"\"\" Load saved model.\n\n    :param savedModelDir: (string)\n           Directory of where the experiment is to be or was saved\n    :returns: (:class:`nupic.frameworks.opf.model.Model`) The loaded model\n              instance.\n    \"\"\"\n    if newSerialization:\n      return HTMPredictionModel.readFromCheckpoint(savedModelDir)\n    else:\n      return Model.load(savedModelDir)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef compute(self, activeColumns, learn=True):\n    self.activateCells(sorted(activeColumns), learn)\n    self.activateDendrites(learn)", "response": "Performs one time step of the Temporal Memory algorithm."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nactivating the active cells using the current active columns and dendrite segments.", "response": "def activateCells(self, activeColumns, learn=True):\n    \"\"\"\n    Calculate the active cells, using the current active columns and dendrite\n    segments. Grow and reinforce synapses.\n\n    :param activeColumns: (iter) A sorted list of active column indices.\n\n    :param learn: (bool) If true, reinforce / punish / grow synapses.\n\n      **Pseudocode:**\n      \n      ::\n\n        for each column\n          if column is active and has active distal dendrite segments\n            call activatePredictedColumn\n          if column is active and doesn't have active distal dendrite segments\n            call burstColumn\n          if column is inactive and has matching distal dendrite segments\n            call punishPredictedColumn\n    \"\"\"\n    prevActiveCells = self.activeCells\n    prevWinnerCells = self.winnerCells\n    self.activeCells = []\n    self.winnerCells = []\n\n    segToCol = lambda segment: int(segment.cell / self.cellsPerColumn)\n    identity = lambda x: x\n\n    for columnData in groupby2(activeColumns, identity,\n                               self.activeSegments, segToCol,\n                               self.matchingSegments, segToCol):\n      (column,\n       activeColumns,\n       columnActiveSegments,\n       columnMatchingSegments) = columnData\n      if activeColumns is not None:\n        if columnActiveSegments is not None:\n          cellsToAdd = self.activatePredictedColumn(column,\n                                                    columnActiveSegments,\n                                                    columnMatchingSegments,\n                                                    prevActiveCells,\n                                                    prevWinnerCells,\n                                                    learn)\n\n          self.activeCells += cellsToAdd\n          self.winnerCells += cellsToAdd\n        else:\n          (cellsToAdd,\n           winnerCell) = self.burstColumn(column,\n                                          columnMatchingSegments,\n                                          prevActiveCells,\n                                          prevWinnerCells,\n                                          learn)\n\n          self.activeCells += cellsToAdd\n          self.winnerCells.append(winnerCell)\n      else:\n        if learn:\n          self.punishPredictedColumn(column,\n                                     columnActiveSegments,\n                                     columnMatchingSegments,\n                                     prevActiveCells,\n                                     prevWinnerCells)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef activateDendrites(self, learn=True):\n    (numActiveConnected,\n     numActivePotential) = self.connections.computeActivity(\n       self.activeCells,\n       self.connectedPermanence)\n\n    activeSegments = (\n      self.connections.segmentForFlatIdx(i)\n      for i in xrange(len(numActiveConnected))\n      if numActiveConnected[i] >= self.activationThreshold\n    )\n\n    matchingSegments = (\n      self.connections.segmentForFlatIdx(i)\n      for i in xrange(len(numActivePotential))\n      if numActivePotential[i] >= self.minThreshold\n    )\n\n    self.activeSegments = sorted(activeSegments,\n                                 key=self.connections.segmentPositionSortKey)\n    self.matchingSegments = sorted(matchingSegments,\n                                   key=self.connections.segmentPositionSortKey)\n    self.numActiveConnectedSynapsesForSegment = numActiveConnected\n    self.numActivePotentialSynapsesForSegment = numActivePotential\n\n    if learn:\n      for segment in self.activeSegments:\n        self.lastUsedIterationForSegment[segment.flatIdx] = self.iteration\n      self.iteration += 1", "response": "Activates the dendrites in the current active cells."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef reset(self):\n    self.activeCells = []\n    self.winnerCells = []\n    self.activeSegments = []\n    self.matchingSegments = []", "response": "Resets the internal state of the sequence to its initial state."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef activatePredictedColumn(self, column, columnActiveSegments,\n                              columnMatchingSegments, prevActiveCells,\n                              prevWinnerCells, learn):\n    \"\"\"\n    Determines which cells in a predicted column should be added to winner cells\n    list, and learns on the segments that correctly predicted this column.\n\n    :param column: (int) Index of bursting column.\n\n    :param columnActiveSegments: (iter) Active segments in this column.\n\n    :param columnMatchingSegments: (iter) Matching segments in this column.\n\n    :param prevActiveCells: (list) Active cells in ``t-1``.\n\n    :param prevWinnerCells: (list) Winner cells in ``t-1``.\n\n    :param learn: (bool) If true, grow and reinforce synapses.\n\n    :returns: (list) A list of predicted cells that will be added to \n              active cells and winner cells.\n    \"\"\"\n    return self._activatePredictedColumn(\n      self.connections, self._random,\n      columnActiveSegments, prevActiveCells, prevWinnerCells,\n      self.numActivePotentialSynapsesForSegment,\n      self.maxNewSynapseCount, self.initialPermanence,\n      self.permanenceIncrement, self.permanenceDecrement,\n      self.maxSynapsesPerSegment, learn)", "response": "Activates a column and returns a list of active cells and winner cells."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\npunishes the Segments that incorrectly predicted a column to be active.", "response": "def punishPredictedColumn(self, column, columnActiveSegments,\n                            columnMatchingSegments, prevActiveCells,\n                            prevWinnerCells):\n    \"\"\"\n    Punishes the Segments that incorrectly predicted a column to be active.\n\n    :param column: (int) Index of bursting column.\n\n    :param columnActiveSegments: (iter) Active segments for this column, or None \n           if there aren't any.\n\n    :param columnMatchingSegments: (iter) Matching segments for this column, or \n           None if there aren't any.\n\n    :param prevActiveCells: (list) Active cells in ``t-1``.\n\n    :param prevWinnerCells: (list) Winner cells in ``t-1``.\n\n    \"\"\"\n    self._punishPredictedColumn(\n      self.connections, columnMatchingSegments, prevActiveCells,\n      self.predictedSegmentDecrement)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef createSegment(self, cell):\n    return self._createSegment(\n      self.connections, self.lastUsedIterationForSegment, cell, self.iteration,\n      self.maxSegmentsPerCell)", "response": "Creates a segment on the specified cell."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _activatePredictedColumn(cls, connections, random, columnActiveSegments,\n                               prevActiveCells, prevWinnerCells,\n                               numActivePotentialSynapsesForSegment,\n                               maxNewSynapseCount, initialPermanence,\n                               permanenceIncrement, permanenceDecrement,\n                               maxSynapsesPerSegment, learn):\n    \"\"\"\n    :param connections: (Object)\n    Connections for the TM. Gets mutated.\n\n    :param random: (Object)\n    Random number generator. Gets mutated.\n\n    :param columnActiveSegments: (iter)\n    Active segments in this column.\n\n    :param prevActiveCells: (list)\n    Active cells in `t-1`.\n\n    :param prevWinnerCells: (list)\n    Winner cells in `t-1`.\n\n    :param numActivePotentialSynapsesForSegment: (list)\n    Number of active potential synapses per segment, indexed by the segment's\n    flatIdx.\n\n    :param maxNewSynapseCount: (int)\n    The maximum number of synapses added to a segment during learning\n\n    :param initialPermanence: (float)\n    Initial permanence of a new synapse.\n\n    @permanenceIncrement (float)\n    Amount by which permanences of synapses are incremented during learning.\n\n    @permanenceDecrement (float)\n    Amount by which permanences of synapses are decremented during learning.\n\n    :param maxSynapsesPerSegment: (int)\n    The maximum number of synapses per segment.\n\n    :param learn: (bool)\n    If true, grow and reinforce synapses.\n\n    :returns: cellsToAdd (list)\n    A list of predicted cells that will be added to active cells and winner\n    cells.\n\n    Pseudocode:\n    for each cell in the column that has an active distal dendrite segment\n      mark the cell as active\n      mark the cell as a winner cell\n      (learning) for each active distal dendrite segment\n        strengthen active synapses\n        weaken inactive synapses\n        grow synapses to previous winner cells\n    \"\"\"\n\n    cellsToAdd = []\n    previousCell = None\n    for segment in columnActiveSegments:\n      if segment.cell != previousCell:\n        cellsToAdd.append(segment.cell)\n        previousCell = segment.cell\n\n      if learn:\n        cls._adaptSegment(connections, segment, prevActiveCells,\n                          permanenceIncrement, permanenceDecrement)\n\n        active = numActivePotentialSynapsesForSegment[segment.flatIdx]\n        nGrowDesired = maxNewSynapseCount - active\n\n        if nGrowDesired > 0:\n          cls._growSynapses(connections, random, segment, nGrowDesired,\n                            prevWinnerCells, initialPermanence,\n                            maxSynapsesPerSegment)\n\n    return cellsToAdd", "response": "Activates the predicted column."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _burstColumn(cls, connections, random, lastUsedIterationForSegment,\n                   column, columnMatchingSegments, prevActiveCells,\n                   prevWinnerCells, cellsForColumn,\n                   numActivePotentialSynapsesForSegment, iteration,\n                   maxNewSynapseCount, initialPermanence, permanenceIncrement,\n                   permanenceDecrement, maxSegmentsPerCell,\n                   maxSynapsesPerSegment, learn):\n    \"\"\"\n    :param connections: (Object)\n    Connections for the TM. Gets mutated.\n\n    :param random: (Object)\n    Random number generator. Gets mutated.\n\n    :param lastUsedIterationForSegment: (list)\n    Last used iteration for each segment, indexed by the segment's flatIdx.\n    Gets mutated.\n\n    :param column: (int)\n    Index of bursting column.\n\n    :param columnMatchingSegments: (iter)\n    Matching segments in this column.\n\n    :param prevActiveCells: (list)\n    Active cells in `t-1`.\n\n    :param prevWinnerCells: (list)\n    Winner cells in `t-1`.\n\n    :param cellsForColumn: (sequence)\n    Range of cell indices on which to operate.\n\n    :param numActivePotentialSynapsesForSegment: (list)\n    Number of active potential synapses per segment, indexed by the segment's\n    flatIdx.\n\n    :param iteration: (int)\n    The current timestep.\n\n    :param maxNewSynapseCount: (int)\n    The maximum number of synapses added to a segment during learning.\n\n    :param initialPermanence: (float)\n    Initial permanence of a new synapse.\n\n    :param permanenceIncrement: (float)\n    Amount by which permanences of synapses are incremented during learning.\n\n    :param permanenceDecrement: (float)\n    Amount by which permanences of synapses are decremented during learning.\n\n    :param maxSegmentsPerCell: (int)\n    The maximum number of segments per cell.\n\n    :param maxSynapsesPerSegment: (int)\n    The maximum number of synapses per segment.\n\n    :param learn: (bool)\n    Whether or not learning is enabled.\n\n    :returns: (tuple) Contains:\n                      `cells`         (iter),\n                      `winnerCell`    (int),\n\n    Pseudocode:\n    mark all cells as active\n    if there are any matching distal dendrite segments\n      find the most active matching segment\n      mark its cell as a winner cell\n      (learning)\n        grow and reinforce synapses to previous winner cells\n    else\n      find the cell with the least segments, mark it as a winner cell\n      (learning)\n        (optimization) if there are prev winner cells\n          add a segment to this winner cell\n          grow synapses to previous winner cells\n    \"\"\"\n    if columnMatchingSegments is not None:\n      numActive = lambda s: numActivePotentialSynapsesForSegment[s.flatIdx]\n      bestMatchingSegment = max(columnMatchingSegments, key=numActive)\n      winnerCell = bestMatchingSegment.cell\n\n      if learn:\n        cls._adaptSegment(connections, bestMatchingSegment, prevActiveCells,\n                          permanenceIncrement, permanenceDecrement)\n\n        nGrowDesired = maxNewSynapseCount - numActive(bestMatchingSegment)\n\n        if nGrowDesired > 0:\n          cls._growSynapses(connections, random, bestMatchingSegment,\n                            nGrowDesired, prevWinnerCells, initialPermanence,\n                            maxSynapsesPerSegment)\n    else:\n      winnerCell = cls._leastUsedCell(random, cellsForColumn, connections)\n      if learn:\n        nGrowExact = min(maxNewSynapseCount, len(prevWinnerCells))\n        if nGrowExact > 0:\n          segment = cls._createSegment(connections,\n                                       lastUsedIterationForSegment, winnerCell,\n                                       iteration, maxSegmentsPerCell)\n          cls._growSynapses(connections, random, segment, nGrowExact,\n                            prevWinnerCells, initialPermanence,\n                            maxSynapsesPerSegment)\n\n    return cellsForColumn, winnerCell", "response": "This method is used to create a bursting column."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\npunishing the predicted column.", "response": "def _punishPredictedColumn(cls, connections, columnMatchingSegments,\n                             prevActiveCells, predictedSegmentDecrement):\n    \"\"\"\n    :param connections: (Object)\n    Connections for the TM. Gets mutated.\n\n    :param columnMatchingSegments: (iter)\n    Matching segments for this column.\n\n    :param prevActiveCells: (list)\n    Active cells in `t-1`.\n\n    :param predictedSegmentDecrement: (float)\n    Amount by which segments are punished for incorrect predictions.\n\n    Pseudocode:\n    for each matching segment in the column\n      weaken active synapses\n    \"\"\"\n    if predictedSegmentDecrement > 0.0 and columnMatchingSegments is not None:\n      for segment in columnMatchingSegments:\n        cls._adaptSegment(connections, segment, prevActiveCells,\n                          -predictedSegmentDecrement, 0.0)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _createSegment(cls, connections, lastUsedIterationForSegment, cell,\n                     iteration, maxSegmentsPerCell):\n    \"\"\"\n    Create a segment on the connections, enforcing the maxSegmentsPerCell\n    parameter.\n    \"\"\"\n    # Enforce maxSegmentsPerCell.\n    while connections.numSegments(cell) >= maxSegmentsPerCell:\n      leastRecentlyUsedSegment = min(\n        connections.segmentsForCell(cell),\n        key=lambda segment : lastUsedIterationForSegment[segment.flatIdx])\n\n      connections.destroySegment(leastRecentlyUsedSegment)\n\n    # Create the segment.\n    segment = connections.createSegment(cell)\n\n    # Do TM-specific bookkeeping for the segment.\n    if segment.flatIdx == len(lastUsedIterationForSegment):\n      lastUsedIterationForSegment.append(iteration)\n    elif segment.flatIdx < len(lastUsedIterationForSegment):\n      # A flatIdx was recycled.\n      lastUsedIterationForSegment[segment.flatIdx] = iteration\n    else:\n      raise AssertionError(\n        \"All segments should be created with the TM createSegment method.\")\n\n    return segment", "response": "Create a segment on the connections and enforcing the maxSegmentsPerCell parameter."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _destroyMinPermanenceSynapses(cls, connections, random, segment,\n                                    nDestroy, excludeCells):\n    \"\"\"\n    Destroy nDestroy synapses on the specified segment, but don't destroy\n    synapses to the \"excludeCells\".\n    \"\"\"\n\n    destroyCandidates = sorted(\n      (synapse for synapse in connections.synapsesForSegment(segment)\n       if synapse.presynapticCell not in excludeCells),\n      key=lambda s: s._ordinal\n    )\n\n    for _ in xrange(nDestroy):\n      if len(destroyCandidates) == 0:\n        break\n\n      minSynapse = None\n      minPermanence = float(\"inf\")\n\n      for synapse in destroyCandidates:\n        if synapse.permanence < minPermanence - EPSILON:\n          minSynapse = synapse\n          minPermanence = synapse.permanence\n\n      connections.destroySynapse(minSynapse)\n      destroyCandidates.remove(minSynapse)", "response": "Destroy nDestroy synapses on the specified segment but don t destroy synapses to the excludeCells."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the cell with the smallest number of segments.", "response": "def _leastUsedCell(cls, random, cells, connections):\n    \"\"\"\n    Gets the cell with the smallest number of segments.\n    Break ties randomly.\n\n    :param random: (Object)\n    Random number generator. Gets mutated.\n\n    :param cells: (list)\n    Indices of cells.\n\n    :param connections: (Object)\n    Connections instance for the TM.\n\n    :returns: (int) Cell index.\n    \"\"\"\n    leastUsedCells = []\n    minNumSegments = float(\"inf\")\n    for cell in cells:\n      numSegments = connections.numSegments(cell)\n\n      if numSegments < minNumSegments:\n        minNumSegments = numSegments\n        leastUsedCells = []\n\n      if numSegments == minNumSegments:\n        leastUsedCells.append(cell)\n\n    i = random.getUInt32(len(leastUsedCells))\n    return leastUsedCells[i]"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngrows synapses on the segment.", "response": "def _growSynapses(cls, connections, random, segment, nDesiredNewSynapes,\n                    prevWinnerCells, initialPermanence, maxSynapsesPerSegment):\n    \"\"\"\n    Creates nDesiredNewSynapes synapses on the segment passed in if\n    possible, choosing random cells from the previous winner cells that are\n    not already on the segment.\n\n    :param connections:        (Object) Connections instance for the tm\n    :param random:             (Object) TM object used to generate random\n                                        numbers\n    :param segment:            (int)    Segment to grow synapses on.\n    :param nDesiredNewSynapes: (int)    Desired number of synapses to grow\n    :param prevWinnerCells:    (list)   Winner cells in `t-1`\n    :param initialPermanence:  (float)  Initial permanence of a new synapse.\n\n    \"\"\"\n    candidates = list(prevWinnerCells)\n\n    for synapse in connections.synapsesForSegment(segment):\n      i = binSearch(candidates, synapse.presynapticCell)\n      if i != -1:\n        del candidates[i]\n\n    nActual = min(nDesiredNewSynapes, len(candidates))\n\n    # Check if we're going to surpass the maximum number of synapses.\n    overrun = connections.numSynapses(segment) + nActual - maxSynapsesPerSegment\n    if overrun > 0:\n      cls._destroyMinPermanenceSynapses(connections, random, segment, overrun,\n                                        prevWinnerCells)\n\n    # Recalculate in case we weren't able to destroy as many synapses as needed.\n    nActual = min(nActual,\n                  maxSynapsesPerSegment - connections.numSynapses(segment))\n\n    for _ in range(nActual):\n      i = random.getUInt32(len(candidates))\n      connections.createSynapse(segment, candidates[i], initialPermanence)\n      del candidates[i]"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _adaptSegment(cls, connections, segment, prevActiveCells,\n                    permanenceIncrement, permanenceDecrement):\n    \"\"\"\n    Updates synapses on segment.\n    Strengthens active synapses; weakens inactive synapses.\n\n    :param connections:          (Object) Connections instance for the tm\n    :param segment:              (int)    Segment to adapt\n    :param prevActiveCells:      (list)   Active cells in `t-1`\n    :param permanenceIncrement:  (float)  Amount to increment active synapses\n    :param permanenceDecrement:  (float)  Amount to decrement inactive synapses\n    \"\"\"\n\n    # Destroying a synapse modifies the set that we're iterating through.\n    synapsesToDestroy = []\n\n    for synapse in connections.synapsesForSegment(segment):\n      permanence = synapse.permanence\n\n      if binSearch(prevActiveCells, synapse.presynapticCell) != -1:\n        permanence += permanenceIncrement\n      else:\n        permanence -= permanenceDecrement\n\n      # Keep permanence within min/max bounds\n      permanence = max(0.0, min(1.0, permanence))\n\n      if permanence < EPSILON:\n        synapsesToDestroy.append(synapse)\n      else:\n        connections.updateSynapsePermanence(synapse, permanence)\n\n    for synapse in synapsesToDestroy:\n      connections.destroySynapse(synapse)\n\n    if connections.numSynapses(segment) == 0:\n      connections.destroySegment(segment)", "response": "Adaptes the given segment to the next active synapses and weakens inactive synapses."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the index of the column that a cell belongs to.", "response": "def columnForCell(self, cell):\n    \"\"\"\n    Returns the index of the column that a cell belongs to.\n\n    :param cell: (int) Cell index\n\n    :returns: (int) Column index\n    \"\"\"\n    self._validateCell(cell)\n\n    return int(cell / self.cellsPerColumn)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the indices of cells that belong to a column.", "response": "def cellsForColumn(self, column):\n    \"\"\"\n    Returns the indices of cells that belong to a column.\n\n    :param column: (int) Column index\n\n    :returns: (list) Cell indices\n    \"\"\"\n    self._validateColumn(column)\n\n    start = self.cellsPerColumn * column\n    end = start + self.cellsPerColumn\n    return range(start, end)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef mapCellsToColumns(self, cells):\n    cellsForColumns = defaultdict(set)\n\n    for cell in cells:\n      column = self.columnForCell(cell)\n      cellsForColumns[column].add(cell)\n\n    return cellsForColumns", "response": "Maps cells to the columns they belong to."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the indices of the predictive cells.", "response": "def getPredictiveCells(self):\n    \"\"\" Returns the indices of the predictive cells.\n\n    :returns: (list) Indices of predictive cells.\n    \"\"\"\n    previousCell = None\n    predictiveCells = []\n    for segment in self.activeSegments:\n      if segment.cell != previousCell:\n        predictiveCells.append(segment.cell)\n        previousCell = segment.cell\n\n    return predictiveCells"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef write(self, proto):\n    # capnp fails to save a tuple.  Let's force columnDimensions to list.\n    proto.columnDimensions = list(self.columnDimensions)\n    proto.cellsPerColumn = self.cellsPerColumn\n    proto.activationThreshold = self.activationThreshold\n    proto.initialPermanence = round(self.initialPermanence, EPSILON_ROUND)\n    proto.connectedPermanence = round(self.connectedPermanence, EPSILON_ROUND)\n    proto.minThreshold = self.minThreshold\n    proto.maxNewSynapseCount = self.maxNewSynapseCount\n    proto.permanenceIncrement = round(self.permanenceIncrement, EPSILON_ROUND)\n    proto.permanenceDecrement = round(self.permanenceDecrement, EPSILON_ROUND)\n    proto.predictedSegmentDecrement = self.predictedSegmentDecrement\n\n    proto.maxSegmentsPerCell = self.maxSegmentsPerCell\n    proto.maxSynapsesPerSegment = self.maxSynapsesPerSegment\n\n    self.connections.write(proto.connections)\n    self._random.write(proto.random)\n\n    proto.activeCells = list(self.activeCells)\n    proto.winnerCells = list(self.winnerCells)\n\n    protoActiveSegments = proto.init(\"activeSegments\", len(self.activeSegments))\n    for i, segment in enumerate(self.activeSegments):\n      protoActiveSegments[i].cell = segment.cell\n      idx = self.connections.segmentsForCell(segment.cell).index(segment)\n      protoActiveSegments[i].idxOnCell = idx\n\n    protoMatchingSegments = proto.init(\"matchingSegments\",\n                                       len(self.matchingSegments))\n    for i, segment in enumerate(self.matchingSegments):\n      protoMatchingSegments[i].cell = segment.cell\n      idx = self.connections.segmentsForCell(segment.cell).index(segment)\n      protoMatchingSegments[i].idxOnCell = idx\n\n    protoNumActivePotential = proto.init(\n      \"numActivePotentialSynapsesForSegment\",\n      len(self.numActivePotentialSynapsesForSegment))\n    for i, numActivePotentialSynapses in enumerate(\n        self.numActivePotentialSynapsesForSegment):\n      segment = self.connections.segmentForFlatIdx(i)\n      if segment is not None:\n        protoNumActivePotential[i].cell = segment.cell\n        idx = self.connections.segmentsForCell(segment.cell).index(segment)\n        protoNumActivePotential[i].idxOnCell = idx\n        protoNumActivePotential[i].number = numActivePotentialSynapses\n\n    proto.iteration = self.iteration\n\n    protoLastUsedIteration = proto.init(\n      \"lastUsedIterationForSegment\",\n      len(self.numActivePotentialSynapsesForSegment))\n    for i, lastUsed in enumerate(self.lastUsedIterationForSegment):\n      segment = self.connections.segmentForFlatIdx(i)\n      if segment is not None:\n        protoLastUsedIteration[i].cell = segment.cell\n        idx = self.connections.segmentsForCell(segment.cell).index(segment)\n        protoLastUsedIteration[i].idxOnCell = idx\n        protoLastUsedIteration[i].number = lastUsed", "response": "Writes serialized data to proto object."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef read(cls, proto):\n    tm = object.__new__(cls)\n\n    # capnp fails to save a tuple, so proto.columnDimensions was forced to\n    # serialize as a list.  We prefer a tuple, however, because columnDimensions\n    # should be regarded as immutable.\n    tm.columnDimensions = tuple(proto.columnDimensions)\n    tm.cellsPerColumn = int(proto.cellsPerColumn)\n    tm.activationThreshold = int(proto.activationThreshold)\n    tm.initialPermanence = round(proto.initialPermanence, EPSILON_ROUND)\n    tm.connectedPermanence = round(proto.connectedPermanence, EPSILON_ROUND)\n    tm.minThreshold = int(proto.minThreshold)\n    tm.maxNewSynapseCount = int(proto.maxNewSynapseCount)\n    tm.permanenceIncrement = round(proto.permanenceIncrement, EPSILON_ROUND)\n    tm.permanenceDecrement = round(proto.permanenceDecrement, EPSILON_ROUND)\n    tm.predictedSegmentDecrement = round(proto.predictedSegmentDecrement,\n                                         EPSILON_ROUND)\n\n    tm.maxSegmentsPerCell = int(proto.maxSegmentsPerCell)\n    tm.maxSynapsesPerSegment = int(proto.maxSynapsesPerSegment)\n\n    tm.connections = Connections.read(proto.connections)\n    #pylint: disable=W0212\n    tm._random = Random()\n    tm._random.read(proto.random)\n    #pylint: enable=W0212\n\n    tm.activeCells = [int(x) for x in proto.activeCells]\n    tm.winnerCells = [int(x) for x in proto.winnerCells]\n\n    flatListLength = tm.connections.segmentFlatListLength()\n    tm.numActiveConnectedSynapsesForSegment = [0] * flatListLength\n    tm.numActivePotentialSynapsesForSegment = [0] * flatListLength\n    tm.lastUsedIterationForSegment = [0] * flatListLength\n\n    tm.activeSegments = []\n    tm.matchingSegments = []\n\n    for protoSegment in proto.activeSegments:\n      tm.activeSegments.append(\n        tm.connections.getSegment(protoSegment.cell,\n                                  protoSegment.idxOnCell))\n\n    for protoSegment in proto.matchingSegments:\n      tm.matchingSegments.append(\n        tm.connections.getSegment(protoSegment.cell,\n                                  protoSegment.idxOnCell))\n\n    for protoSegment in proto.numActivePotentialSynapsesForSegment:\n      segment = tm.connections.getSegment(protoSegment.cell,\n                                          protoSegment.idxOnCell)\n\n      tm.numActivePotentialSynapsesForSegment[segment.flatIdx] = (\n        int(protoSegment.number))\n\n    tm.iteration = long(proto.iteration)\n\n    for protoSegment in proto.lastUsedIterationForSegment:\n      segment = tm.connections.getSegment(protoSegment.cell,\n                                          protoSegment.idxOnCell)\n\n      tm.lastUsedIterationForSegment[segment.flatIdx] = (\n        long(protoSegment.number))\n\n    return tm", "response": "Reads deserialized data from proto object."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef generateFromNumbers(self, numbers):\n    sequence = []\n\n    for number in numbers:\n      if number == None:\n        sequence.append(number)\n      else:\n        pattern = self.patternMachine.get(number)\n        sequence.append(pattern)\n\n    return sequence", "response": "Generate a sequence from a list of numbers."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadd spatial noise to each pattern in the sequence.", "response": "def addSpatialNoise(self, sequence, amount):\n    \"\"\"\n    Add spatial noise to each pattern in the sequence.\n\n    @param sequence (list)  Sequence\n    @param amount   (float) Amount of spatial noise\n\n    @return (list) Sequence with spatial noise\n    \"\"\"\n    newSequence = []\n\n    for pattern in sequence:\n      if pattern is not None:\n        pattern = self.patternMachine.addNoise(pattern, amount)\n      newSequence.append(pattern)\n\n    return newSequence"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute ROC curve for a Receiver operating characteristic.", "response": "def ROCCurve(y_true, y_score):\n    \"\"\"compute Receiver operating characteristic (ROC)\n\n    Note: this implementation is restricted to the binary classification task.\n\n    Parameters\n    ----------\n\n    y_true : array, shape = [n_samples]\n        true binary labels\n\n    y_score : array, shape = [n_samples]\n        target scores, can either be probability estimates of\n        the positive class, confidence values, or binary decisions.\n\n    Returns\n    -------\n    fpr : array, shape = [>2]\n        False Positive Rates\n\n    tpr : array, shape = [>2]\n        True Positive Rates\n\n    thresholds : array, shape = [>2]\n        Thresholds on y_score used to compute fpr and tpr\n\n    Examples\n    --------\n    >>> import numpy as np\n    >>> from sklearn import metrics\n    >>> y = np.array([1, 1, 2, 2])\n    >>> scores = np.array([0.1, 0.4, 0.35, 0.8])\n    >>> fpr, tpr, thresholds = metrics.roc_curve(y, scores)\n    >>> fpr\n    array([ 0. ,  0.5,  0.5,  1. ])\n\n    References\n    ----------\n    http://en.wikipedia.org/wiki/Receiver_operating_characteristic\n\n    \"\"\"\n    y_true = np.ravel(y_true)\n    classes = np.unique(y_true)\n\n    # ROC only for binary classification\n    if classes.shape[0] != 2:\n        raise ValueError(\"ROC is defined for binary classification only\")\n\n    y_score = np.ravel(y_score)\n\n    n_pos = float(np.sum(y_true == classes[1]))  # nb of true positive\n    n_neg = float(np.sum(y_true == classes[0]))  # nb of true negative\n\n    thresholds = np.unique(y_score)\n    neg_value, pos_value = classes[0], classes[1]\n\n    tpr = np.empty(thresholds.size, dtype=np.float)  # True positive rate\n    fpr = np.empty(thresholds.size, dtype=np.float)  # False positive rate\n\n    # Build tpr/fpr vector\n    current_pos_count = current_neg_count = sum_pos = sum_neg = idx = 0\n\n    signal = np.c_[y_score, y_true]\n    sorted_signal = signal[signal[:, 0].argsort(), :][::-1]\n    last_score = sorted_signal[0][0]\n    for score, value in sorted_signal:\n        if score == last_score:\n            if value == pos_value:\n                current_pos_count += 1\n            else:\n                current_neg_count += 1\n        else:\n            tpr[idx] = (sum_pos + current_pos_count) / n_pos\n            fpr[idx] = (sum_neg + current_neg_count) / n_neg\n            sum_pos += current_pos_count\n            sum_neg += current_neg_count\n            current_pos_count = 1 if value == pos_value else 0\n            current_neg_count = 1 if value == neg_value else 0\n            idx += 1\n            last_score = score\n    else:\n        tpr[-1] = (sum_pos + current_pos_count) / n_pos\n        fpr[-1] = (sum_neg + current_neg_count) / n_neg\n\n    # hard decisions, add (0,0)\n    if fpr.shape[0] == 2:\n        fpr = np.array([0.0, fpr[0], fpr[1]])\n        tpr = np.array([0.0, tpr[0], tpr[1]])\n    # trivial decisions, add (0,0) and (1,1)\n    elif fpr.shape[0] == 1:\n        fpr = np.array([0.0, fpr[0], 1.0])\n        tpr = np.array([0.0, tpr[0], 1.0])\n\n    return fpr, tpr, thresholds"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompute the area under curve using the trapezoidal rule", "response": "def AreaUnderCurve(x, y):\n    \"\"\"Compute Area Under the Curve (AUC) using the trapezoidal rule\n\n    Parameters\n    ----------\n    x : array, shape = [n]\n        x coordinates\n\n    y : array, shape = [n]\n        y coordinates\n\n    Returns\n    -------\n    auc : float\n\n    Examples\n    --------\n    >>> import numpy as np\n    >>> from sklearn import metrics\n    >>> y = np.array([1, 1, 2, 2])\n    >>> pred = np.array([0.1, 0.4, 0.35, 0.8])\n    >>> fpr, tpr, thresholds = metrics.roc_curve(y, pred)\n    >>> metrics.auc(fpr, tpr)\n    0.75\n\n    \"\"\"\n    #x, y = check_arrays(x, y)\n    if x.shape[0] != y.shape[0]:\n        raise ValueError('x and y should have the same shape'\n                         ' to compute area under curve,'\n                         ' but x.shape = %s and y.shape = %s.'\n                         % (x.shape, y.shape))\n    if x.shape[0] < 2:\n        raise ValueError('At least 2 points are needed to compute'\n                         ' area under curve, but x.shape = %s' % x.shape)\n\n    # reorder the data points according to the x axis\n    order = np.argsort(x)\n    x = x[order]\n    y = y[order]\n\n    h = np.diff(x)\n    area = np.sum(h * (y[1:] + y[:-1])) / 2.0\n    return area"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef mmPrettyPrintTraces(traces, breakOnResets=None):\n    assert len(traces) > 0, \"No traces found\"\n    table = PrettyTable([\"#\"] + [trace.prettyPrintTitle() for trace in traces])\n\n    for i in xrange(len(traces[0].data)):\n      if breakOnResets and breakOnResets.data[i]:\n        table.add_row([\"<reset>\"] * (len(traces) + 1))\n      table.add_row([i] +\n        [trace.prettyPrintDatum(trace.data[i]) for trace in traces])\n\n    return table.get_string().encode(\"utf-8\")", "response": "Pretty - print a list of traces in a node."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn pretty-printed table of metrics. @param metrics (list) Traces to print in table @param sigFigs (int) Number of significant figures to print @return (string) Pretty-printed table of metrics.", "response": "def mmPrettyPrintMetrics(metrics, sigFigs=5):\n    \"\"\"\n    Returns pretty-printed table of metrics.\n\n    @param metrics (list) Traces to print in table\n    @param sigFigs (int)  Number of significant figures to print\n\n    @return (string) Pretty-printed table of metrics.\n    \"\"\"\n    assert len(metrics) > 0, \"No metrics found\"\n    table = PrettyTable([\"Metric\", \"mean\", \"standard deviation\",\n                         \"min\", \"max\", \"sum\", ])\n\n    for metric in metrics:\n      table.add_row([metric.prettyPrintTitle()] + metric.getStats())\n\n    return table.get_string().encode(\"utf-8\")"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef mmGetCellTracePlot(self, cellTrace, cellCount, activityType, title=\"\",\n                         showReset=False, resetShading=0.25):\n    \"\"\"\n    Returns plot of the cell activity. Note that if many timesteps of\n    activities are input, matplotlib's image interpolation may omit activities\n    (columns in the image).\n\n    @param cellTrace    (list)   a temporally ordered list of sets of cell\n                                 activities\n\n    @param cellCount    (int)    number of cells in the space being rendered\n\n    @param activityType (string) type of cell activity being displayed\n\n    @param title        (string) an optional title for the figure\n\n    @param showReset    (bool)   if true, the first set of cell activities\n                                 after a reset will have a grayscale background\n\n    @param resetShading (float)  applicable if showReset is true, specifies the\n                                 intensity of the reset background with 0.0\n                                 being white and 1.0 being black\n\n    @return (Plot) plot\n    \"\"\"\n    plot = Plot(self, title)\n    resetTrace = self.mmGetTraceResets().data\n    data = numpy.zeros((cellCount, 1))\n    for i in xrange(len(cellTrace)):\n      # Set up a \"background\" vector that is shaded or blank\n      if showReset and resetTrace[i]:\n        activity = numpy.ones((cellCount, 1)) * resetShading\n      else:\n        activity = numpy.zeros((cellCount, 1))\n\n      activeIndices = cellTrace[i]\n      activity[list(activeIndices)] = 1\n      data = numpy.concatenate((data, activity), 1)\n\n    plot.add2DArray(data, xlabel=\"Time\", ylabel=activityType, name=title)\n    return plot", "response": "Returns a matplotlib plot of the cell activity."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ngive a series of anomaly scores, compute the likelihood for each score. This function should be called once on a bunch of historical anomaly scores for an initial estimate of the distribution. It should be called again every so often (say every 50 records) to update the estimate. :param anomalyScores: a list of records. Each record is a list with the following three elements: [timestamp, value, score] Example:: [datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0] For best results, the list should be between 1000 and 10,000 records :param averagingWindow: integer number of records to average over :param skipRecords: integer specifying number of records to skip when estimating distributions. If skip records are >= len(anomalyScores), a very broad distribution is returned that makes everything pretty likely. :param verbosity: integer controlling extent of printouts for debugging 0 = none 1 = occasional information 2 = print every record :returns: 3-tuple consisting of: - likelihoods numpy array of likelihoods, one for each aggregated point - avgRecordList list of averaged input records - params a small JSON dict that contains the state of the estimator", "response": "def estimateAnomalyLikelihoods(anomalyScores,\n                               averagingWindow=10,\n                               skipRecords=0,\n                               verbosity=0):\n  \"\"\"\n  Given a series of anomaly scores, compute the likelihood for each score. This\n  function should be called once on a bunch of historical anomaly scores for an\n  initial estimate of the distribution. It should be called again every so often\n  (say every 50 records) to update the estimate.\n\n  :param anomalyScores: a list of records. Each record is a list with the\n                        following three elements: [timestamp, value, score]\n\n                        Example::\n\n                            [datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]\n\n                        For best results, the list should be between 1000\n                        and 10,000 records\n  :param averagingWindow: integer number of records to average over\n  :param skipRecords: integer specifying number of records to skip when\n                      estimating distributions. If skip records are >=\n                      len(anomalyScores), a very broad distribution is returned\n                      that makes everything pretty likely.\n  :param verbosity: integer controlling extent of printouts for debugging\n\n                      0 = none\n                      1 = occasional information\n                      2 = print every record\n\n  :returns: 3-tuple consisting of:\n\n            - likelihoods\n\n              numpy array of likelihoods, one for each aggregated point\n\n            - avgRecordList\n\n              list of averaged input records\n\n            - params\n\n              a small JSON dict that contains the state of the estimator\n\n  \"\"\"\n  if verbosity > 1:\n    print(\"In estimateAnomalyLikelihoods.\")\n    print(\"Number of anomaly scores:\", len(anomalyScores))\n    print(\"Skip records=\", skipRecords)\n    print(\"First 20:\", anomalyScores[0:min(20, len(anomalyScores))])\n\n  if len(anomalyScores) == 0:\n    raise ValueError(\"Must have at least one anomalyScore\")\n\n  # Compute averaged anomaly scores\n  aggRecordList, historicalValues, total =  _anomalyScoreMovingAverage(\n    anomalyScores,\n    windowSize = averagingWindow,\n    verbosity = verbosity)\n  s = [r[2] for r in aggRecordList]\n  dataValues = numpy.array(s)\n\n  # Estimate the distribution of anomaly scores based on aggregated records\n  if len(aggRecordList) <= skipRecords:\n    distributionParams = nullDistribution(verbosity = verbosity)\n  else:\n    distributionParams = estimateNormal(dataValues[skipRecords:])\n\n    # HACK ALERT! The HTMPredictionModel currently does not handle constant\n    # metric values very well (time of day encoder changes sometimes lead to\n    # unstable SDR's even though the metric is constant). Until this is\n    # resolved, we explicitly detect and handle completely flat metric values by\n    # reporting them as not anomalous.\n    s = [r[1] for r in aggRecordList]\n    # Only do this if the values are numeric\n    if all([isinstance(r[1], numbers.Number) for r in aggRecordList]):\n      metricValues = numpy.array(s)\n      metricDistribution = estimateNormal(metricValues[skipRecords:],\n                                          performLowerBoundCheck=False)\n\n      if metricDistribution[\"variance\"] < 1.5e-5:\n        distributionParams = nullDistribution(verbosity = verbosity)\n\n  # Estimate likelihoods based on this distribution\n  likelihoods = numpy.array(dataValues, dtype=float)\n  for i, s in enumerate(dataValues):\n    likelihoods[i] = tailProbability(s, distributionParams)\n\n  # Filter likelihood values\n  filteredLikelihoods = numpy.array(\n    _filterLikelihoods(likelihoods) )\n\n  params = {\n    \"distribution\":       distributionParams,\n    \"movingAverage\": {\n      \"historicalValues\": historicalValues,\n      \"total\":            total,\n      \"windowSize\":       averagingWindow,\n    },\n    \"historicalLikelihoods\":\n      list(likelihoods[-min(averagingWindow, len(likelihoods)):]),\n  }\n\n  if verbosity > 1:\n    print(\"Discovered params=\")\n    print(params)\n    print(\"Number of likelihoods:\", len(likelihoods))\n    print(\"First 20 likelihoods:\", (\n      filteredLikelihoods[0:min(20, len(filteredLikelihoods))] ))\n    print(\"leaving estimateAnomalyLikelihoods\")\n\n\n  return (filteredLikelihoods, aggRecordList, params)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nupdating the anomaly likelihoods for a set of anomaly scores using the given parameters.", "response": "def updateAnomalyLikelihoods(anomalyScores,\n                             params,\n                             verbosity=0):\n  \"\"\"\n  Compute updated probabilities for anomalyScores using the given params.\n\n  :param anomalyScores: a list of records. Each record is a list with the\n                        following three elements: [timestamp, value, score]\n\n                        Example::\n\n                            [datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]\n\n  :param params: the JSON dict returned by estimateAnomalyLikelihoods\n  :param verbosity: integer controlling extent of printouts for debugging\n  :type verbosity: int\n\n  :returns: 3-tuple consisting of:\n\n            - likelihoods\n\n              numpy array of likelihoods, one for each aggregated point\n\n            - avgRecordList\n\n              list of averaged input records\n\n            - params\n\n              an updated JSON object containing the state of this metric.\n\n  \"\"\"\n  if verbosity > 3:\n    print(\"In updateAnomalyLikelihoods.\")\n    print(\"Number of anomaly scores:\", len(anomalyScores))\n    print(\"First 20:\", anomalyScores[0:min(20, len(anomalyScores))])\n    print(\"Params:\", params)\n\n  if len(anomalyScores) == 0:\n    raise ValueError(\"Must have at least one anomalyScore\")\n\n  if not isValidEstimatorParams(params):\n    raise ValueError(\"'params' is not a valid params structure\")\n\n  # For backward compatibility.\n  if \"historicalLikelihoods\" not in params:\n    params[\"historicalLikelihoods\"] = [1.0]\n\n  # Compute moving averages of these new scores using the previous values\n  # as well as likelihood for these scores using the old estimator\n  historicalValues  = params[\"movingAverage\"][\"historicalValues\"]\n  total             = params[\"movingAverage\"][\"total\"]\n  windowSize        = params[\"movingAverage\"][\"windowSize\"]\n\n  aggRecordList = numpy.zeros(len(anomalyScores), dtype=float)\n  likelihoods = numpy.zeros(len(anomalyScores), dtype=float)\n  for i, v in enumerate(anomalyScores):\n    newAverage, historicalValues, total = (\n      MovingAverage.compute(historicalValues, total, v[2], windowSize)\n    )\n    aggRecordList[i] = newAverage\n    likelihoods[i]   = tailProbability(newAverage, params[\"distribution\"])\n\n  # Filter the likelihood values. First we prepend the historical likelihoods\n  # to the current set. Then we filter the values.  We peel off the likelihoods\n  # to return and the last windowSize values to store for later.\n  likelihoods2 = params[\"historicalLikelihoods\"] + list(likelihoods)\n  filteredLikelihoods = _filterLikelihoods(likelihoods2)\n  likelihoods[:] = filteredLikelihoods[-len(likelihoods):]\n  historicalLikelihoods = likelihoods2[-min(windowSize, len(likelihoods2)):]\n\n  # Update the estimator\n  newParams = {\n    \"distribution\": params[\"distribution\"],\n    \"movingAverage\": {\n      \"historicalValues\": historicalValues,\n      \"total\": total,\n      \"windowSize\": windowSize,\n    },\n    \"historicalLikelihoods\": historicalLikelihoods,\n  }\n\n  assert len(newParams[\"historicalLikelihoods\"]) <= windowSize\n\n  if verbosity > 3:\n    print(\"Number of likelihoods:\", len(likelihoods))\n    print(\"First 20 likelihoods:\", likelihoods[0:min(20, len(likelihoods))])\n    print(\"Leaving updateAnomalyLikelihoods.\")\n\n  return (likelihoods, aggRecordList, newParams)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nfilters the likelihoods in a list of floats so that we only preserve the likelihoods in likelihood.", "response": "def _filterLikelihoods(likelihoods,\n                       redThreshold=0.99999, yellowThreshold=0.999):\n  \"\"\"\n  Filter the list of raw (pre-filtered) likelihoods so that we only preserve\n  sharp increases in likelihood. 'likelihoods' can be a numpy array of floats or\n  a list of floats.\n\n  :returns: A new list of floats likelihoods containing the filtered values.\n  \"\"\"\n  redThreshold    = 1.0 - redThreshold\n  yellowThreshold = 1.0 - yellowThreshold\n\n  # The first value is untouched\n  filteredLikelihoods = [likelihoods[0]]\n\n  for i, v in enumerate(likelihoods[1:]):\n\n    if v <= redThreshold:\n      # Value is in the redzone\n\n      if likelihoods[i] > redThreshold:\n        # Previous value is not in redzone, so leave as-is\n        filteredLikelihoods.append(v)\n      else:\n        filteredLikelihoods.append(yellowThreshold)\n\n    else:\n      # Value is below the redzone, so leave as-is\n      filteredLikelihoods.append(v)\n\n  return filteredLikelihoods"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngiving a list of anomaly scores return a list of averaged records. anomalyScores is assumed to be a list of records of the form: [datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0] Each record in the returned list list contains: [datetime, value, averagedScore] *Note:* we only average the anomaly score.", "response": "def _anomalyScoreMovingAverage(anomalyScores,\n                               windowSize=10,\n                               verbosity=0,\n                              ):\n  \"\"\"\n  Given a list of anomaly scores return a list of averaged records.\n  anomalyScores is assumed to be a list of records of the form:\n                [datetime.datetime(2013, 8, 10, 23, 0), 6.0, 1.0]\n\n  Each record in the returned list list contains:\n      [datetime, value, averagedScore]\n\n  *Note:* we only average the anomaly score.\n  \"\"\"\n\n  historicalValues = []\n  total = 0.0\n  averagedRecordList = []    # Aggregated records\n  for record in anomalyScores:\n\n    # Skip (but log) records without correct number of entries\n    if not isinstance(record, (list, tuple)) or len(record) != 3:\n      if verbosity >= 1:\n        print(\"Malformed record:\", record)\n      continue\n\n    avg, historicalValues, total = (\n      MovingAverage.compute(historicalValues, total, record[2], windowSize)\n      )\n\n    averagedRecordList.append( [record[0], record[1], avg] )\n\n    if verbosity > 2:\n      print(\"Aggregating input record:\", record)\n      print(\"Result:\", [record[0], record[1], avg])\n\n  return averagedRecordList, historicalValues, total"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nestimating the parameters of a normal distribution based on the sampleData.", "response": "def estimateNormal(sampleData, performLowerBoundCheck=True):\n  \"\"\"\n  :param sampleData:\n  :type sampleData: Numpy array.\n  :param performLowerBoundCheck:\n  :type performLowerBoundCheck: bool\n  :returns: A dict containing the parameters of a normal distribution based on\n      the ``sampleData``.\n  \"\"\"\n  params = {\n    \"name\": \"normal\",\n    \"mean\": numpy.mean(sampleData),\n    \"variance\": numpy.var(sampleData),\n  }\n\n  if performLowerBoundCheck:\n    # Handle edge case of almost no deviations and super low anomaly scores. We\n    # find that such low anomaly means can happen, but then the slightest blip\n    # of anomaly score can cause the likelihood to jump up to red.\n    if params[\"mean\"] < 0.03:\n      params[\"mean\"] = 0.03\n\n    # Catch all for super low variance to handle numerical precision issues\n    if params[\"variance\"] < 0.0003:\n      params[\"variance\"] = 0.0003\n\n  # Compute standard deviation\n  if params[\"variance\"] > 0:\n    params[\"stdev\"] = math.sqrt(params[\"variance\"])\n  else:\n    params[\"stdev\"] = 0\n\n  return params"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef tailProbability(x, distributionParams):\n  if \"mean\" not in distributionParams or \"stdev\" not in distributionParams:\n    raise RuntimeError(\"Insufficient parameters to specify the distribution.\")\n\n  if x < distributionParams[\"mean\"]:\n    # Gaussian is symmetrical around mean, so flip to get the tail probability\n    xp = 2 * distributionParams[\"mean\"] - x\n    return tailProbability(xp, distributionParams)\n\n  # Calculate the Q function with the complementary error function, explained\n  # here: http://www.gaussianwaves.com/2012/07/q-function-and-error-functions\n  z = (x - distributionParams[\"mean\"]) / distributionParams[\"stdev\"]\n  return 0.5 * math.erfc(z/1.4142)", "response": "Calculates the tail probability of getting samples further\n  from the mean and standard deviation of the get - tail - distribution."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchecks if the estimator params are valid.", "response": "def isValidEstimatorParams(p):\n  \"\"\"\n  :returns: ``True`` if ``p`` is a valid estimator params as might be returned\n    by ``estimateAnomalyLikelihoods()`` or ``updateAnomalyLikelihoods``,\n    ``False`` otherwise.  Just does some basic validation.\n  \"\"\"\n  if not isinstance(p, dict):\n    return False\n  if \"distribution\" not in p:\n    return False\n  if \"movingAverage\" not in p:\n    return False\n  dist = p[\"distribution\"]\n  if not (\"mean\" in dist and \"name\" in dist\n          and \"variance\" in dist and \"stdev\" in dist):\n    return False\n\n  return True"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _calcSkipRecords(numIngested, windowSize, learningPeriod):\n    numShiftedOut = max(0, numIngested - windowSize)\n    return min(numIngested, max(0, learningPeriod - numShiftedOut))", "response": "Calculates the value of skipRecords for passing to estimateAnomalyLikelihoods\n   ."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef read(cls, proto):\n    # pylint: disable=W0212\n    anomalyLikelihood = object.__new__(cls)\n    anomalyLikelihood._iteration = proto.iteration\n\n    anomalyLikelihood._historicalScores = collections.deque(\n      maxlen=proto.historicWindowSize)\n    for i, score in enumerate(proto.historicalScores):\n      anomalyLikelihood._historicalScores.append((i, score.value,\n                                                  score.anomalyScore))\n    if proto.distribution.name: # is \"\" when there is no distribution.\n      anomalyLikelihood._distribution = dict()\n      anomalyLikelihood._distribution['distribution'] = dict()\n      anomalyLikelihood._distribution['distribution'][\"name\"] = proto.distribution.name\n      anomalyLikelihood._distribution['distribution'][\"mean\"] = proto.distribution.mean\n      anomalyLikelihood._distribution['distribution'][\"variance\"] = proto.distribution.variance\n      anomalyLikelihood._distribution['distribution'][\"stdev\"] = proto.distribution.stdev\n\n      anomalyLikelihood._distribution[\"movingAverage\"] = {}\n      anomalyLikelihood._distribution[\"movingAverage\"][\"windowSize\"] = proto.distribution.movingAverage.windowSize\n      anomalyLikelihood._distribution[\"movingAverage\"][\"historicalValues\"] = []\n      for value in proto.distribution.movingAverage.historicalValues:\n        anomalyLikelihood._distribution[\"movingAverage\"][\"historicalValues\"].append(value)\n      anomalyLikelihood._distribution[\"movingAverage\"][\"total\"] = proto.distribution.movingAverage.total\n\n      anomalyLikelihood._distribution[\"historicalLikelihoods\"] = []\n      for likelihood in proto.distribution.historicalLikelihoods:\n        anomalyLikelihood._distribution[\"historicalLikelihoods\"].append(likelihood)\n    else:\n      anomalyLikelihood._distribution = None\n\n    anomalyLikelihood._probationaryPeriod = proto.probationaryPeriod\n    anomalyLikelihood._learningPeriod = proto.learningPeriod\n    anomalyLikelihood._reestimationPeriod = proto.reestimationPeriod\n    # pylint: enable=W0212\n\n    return anomalyLikelihood", "response": "This method is used to deserialize the anomaly likelihood object from the given capnp object."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef write(self, proto):\n\n    proto.iteration = self._iteration\n\n    pHistScores = proto.init('historicalScores', len(self._historicalScores))\n    for i, score in enumerate(list(self._historicalScores)):\n      _, value, anomalyScore = score\n      record = pHistScores[i]\n      record.value = float(value)\n      record.anomalyScore = float(anomalyScore)\n\n    if self._distribution:\n      proto.distribution.name = self._distribution[\"distribution\"][\"name\"]\n      proto.distribution.mean = float(self._distribution[\"distribution\"][\"mean\"])\n      proto.distribution.variance = float(self._distribution[\"distribution\"][\"variance\"])\n      proto.distribution.stdev = float(self._distribution[\"distribution\"][\"stdev\"])\n\n      proto.distribution.movingAverage.windowSize = float(self._distribution[\"movingAverage\"][\"windowSize\"])\n\n      historicalValues = self._distribution[\"movingAverage\"][\"historicalValues\"]\n      pHistValues = proto.distribution.movingAverage.init(\n        \"historicalValues\", len(historicalValues))\n      for i, value in enumerate(historicalValues):\n        pHistValues[i] = float(value)\n\n      #proto.distribution.movingAverage.historicalValues = self._distribution[\"movingAverage\"][\"historicalValues\"]\n      proto.distribution.movingAverage.total = float(self._distribution[\"movingAverage\"][\"total\"])\n\n      historicalLikelihoods = self._distribution[\"historicalLikelihoods\"]\n      pHistLikelihoods = proto.distribution.init(\"historicalLikelihoods\",\n                                                 len(historicalLikelihoods))\n      for i, likelihood in enumerate(historicalLikelihoods):\n        pHistLikelihoods[i] = float(likelihood)\n\n    proto.probationaryPeriod = self._probationaryPeriod\n    proto.learningPeriod = self._learningPeriod\n    proto.reestimationPeriod = self._reestimationPeriod\n    proto.historicWindowSize = self._historicalScores.maxlen", "response": "Writes the anomaly likelihood object to the given capnp object."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef anomalyProbability(self, value, anomalyScore, timestamp=None):\n    if timestamp is None:\n      timestamp = self._iteration\n\n    dataPoint = (timestamp, value, anomalyScore)\n    # We ignore the first probationaryPeriod data points\n    if self._iteration < self._probationaryPeriod:\n      likelihood = 0.5\n    else:\n      # On a rolling basis we re-estimate the distribution\n      if ( (self._distribution is None) or\n           (self._iteration % self._reestimationPeriod == 0) ):\n\n        numSkipRecords = self._calcSkipRecords(\n          numIngested=self._iteration,\n          windowSize=self._historicalScores.maxlen,\n          learningPeriod=self._learningPeriod)\n\n        _, _, self._distribution = estimateAnomalyLikelihoods(\n          self._historicalScores,\n          skipRecords=numSkipRecords)\n\n      likelihoods, _, self._distribution = updateAnomalyLikelihoods(\n        [dataPoint],\n        self._distribution)\n\n      likelihood = 1.0 - likelihoods[0]\n\n    # Before we exit update historical scores and iteration\n    self._historicalScores.append(dataPoint)\n    self._iteration += 1\n\n    return likelihood", "response": "Compute the probability that the current value plus anomaly score represents\n    anomaly score."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates and returns the _IterationPhaseLearnOnly instance corresponding to this phase specification model", "response": "def _getImpl(self, model):\n    \"\"\" Creates and returns the _IterationPhase-based instance corresponding\n    to this phase specification\n\n    model:          Model instance\n    \"\"\"\n    impl = _IterationPhaseLearnOnly(model=model,\n                                    nIters=self.__nIters)\n    return impl"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate and returns the _IterationPhaseInferOnly instance corresponding to this phase specification model", "response": "def _getImpl(self, model):\n    \"\"\" Creates and returns the _IterationPhase-based instance corresponding\n    to this phase specification\n\n    model:          Model instance\n    \"\"\"\n    impl = _IterationPhaseInferOnly(model=model,\n                                    nIters=self.__nIters,\n                                    inferenceArgs=self.__inferenceArgs)\n    return impl"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreplaces the Iteration Cycle phases with the given sequence of IterationPhaseSpecXXXXX elements.", "response": "def replaceIterationCycle(self, phaseSpecs):\n    \"\"\" Replaces the Iteration Cycle phases\n\n    :param phaseSpecs: Iteration cycle description consisting of a sequence of\n                  IterationPhaseSpecXXXXX elements that are performed in the\n                  given order\n    \"\"\"\n\n    # -----------------------------------------------------------------------\n    # Replace our phase manager\n    #\n    self.__phaseManager = _PhaseManager(\n      model=self.__model,\n      phaseSpecs=phaseSpecs)\n\n    return"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef handleInputRecord(self, inputRecord):\n    assert inputRecord, \"Invalid inputRecord: %r\" % inputRecord\n\n    results = self.__phaseManager.handleInputRecord(inputRecord)\n    metrics = self.__metricsMgr.update(results)\n\n    # Execute task-postIter callbacks\n    for cb in self.__userCallbacks['postIter']:\n      cb(self.__model)\n\n    results.metrics = metrics\n\n    # Return the input and predictions for this record\n    return results", "response": "Processes the given record according to the current iteration cycle phase\n    Updates the metrics and returns the input and predictions for this record."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef __advancePhase(self):\n    self.__currentPhase = self.__phaseCycler.next()\n    self.__currentPhase.enterPhase()\n\n    return", "response": "Advance to the next iteration cycle phase"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef handleInputRecord(self, inputRecord):\n\n    results = self.__model.run(inputRecord)\n\n    shouldContinue = self.__currentPhase.advance()\n    if not shouldContinue:\n      self.__advancePhase()\n\n    return results", "response": "Processes the given record according to the current phase and returns the results."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing the iterator that is necessary upon entry to the phase.", "response": "def enterPhase(self):\n    \"\"\"\n    Performs initialization that is necessary upon entry to the phase. Must\n    be called before handleInputRecord() at the beginning of each phase\n    \"\"\"\n\n    self.__iter = iter(xrange(self.__nIters))\n\n    # Prime the iterator\n    self.__iter.next()"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nadvance the iteration ; returns True if more iterations remain ; False otherwise.", "response": "def advance(self):\n    \"\"\" Advances the iteration;\n\n    Returns:      True if more iterations remain; False if this is the final\n                  iteration.\n    \"\"\"\n    hasMore = True\n    try:\n      self.__iter.next()\n    except StopIteration:\n      self.__iter = None\n      hasMore = False\n\n    return hasMore"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef enterPhase(self):\n    super(_IterationPhaseLearnOnly, self).enterPhase()\n    self.__model.enableLearning()\n    self.__model.disableInference()\n    return", "response": "Initializes the learning and inference states for the current iteration."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninitializing the inference method that is necessary upon entry to the phase.", "response": "def enterPhase(self):\n    \"\"\" [_IterationPhase method implementation]\n    Performs initialization that is necessary upon entry to the phase. Must\n    be called before handleInputRecord() at the beginning of each phase\n    \"\"\"\n    super(_IterationPhaseInferCommon, self).enterPhase()\n    self._model.enableInference(inferenceArgs=self._inferenceArgs)\n    return"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef write(self, proto):\n    super(PreviousValueModel, self).writeBaseToProto(proto.modelBase)\n\n    proto.fieldNames = self._fieldNames\n    proto.fieldTypes = self._fieldTypes\n    if self._predictedField:\n      proto.predictedField = self._predictedField\n    proto.predictionSteps = self._predictionSteps", "response": "Serialize via capnp\n\n    :param proto: capnp PreviousValueModelProto message builder"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef read(cls, proto):\n    instance = object.__new__(cls)\n    super(PreviousValueModel, instance).__init__(proto=proto.modelBase)\n\n    instance._logger = opf_utils.initLogger(instance)\n    if len(proto.predictedField):\n      instance._predictedField = proto.predictedField\n    else:\n      instance._predictedField = None\n    instance._fieldNames = list(proto.fieldNames)\n    instance._fieldTypes = list(proto.fieldTypes)\n    instance._predictionSteps = list(proto.predictionSteps)\n\n    return instance", "response": "Deserialize via capnp\n\n    :param proto: capnp PreviousValueModelProto message reader\n\n    :returns: new instance of PreviousValueModel deserialized from the given\n              proto"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef lscsum(lx, epsilon=None):\n  lx = numpy.asarray(lx)\n  base = lx.max()\n\n  # If the input is the log of 0's, catch this condition before we generate\n  #  an exception, and return the log(0)\n  if numpy.isinf(base):\n    return base\n\n  # If the user specified an epsilon and we are below it, return epsilon\n  if (epsilon is not None) and (base < epsilon):\n    return epsilon\n\n  x = numpy.exp(lx - base)\n  ssum = x.sum()\n\n  result = numpy.log(ssum) + base\n  # try:\n  #   conventional = numpy.log(numpy.exp(lx).sum())\n  #   if not similar(result, conventional):\n  #     if numpy.isinf(conventional).any() and not numpy.isinf(result).any():\n  #       # print \"Scaled log sum avoided underflow or overflow.\"\n  #       pass\n  #     else:\n  #       import sys\n  #       print >>sys.stderr, \"Warning: scaled log sum did not match.\"\n  #       print >>sys.stderr, \"Scaled log result:\"\n  #       print >>sys.stderr, result\n  #       print >>sys.stderr, \"Conventional result:\"\n  #       print >>sys.stderr, conventional\n  # except FloatingPointError, e:\n  #   # print \"Scaled log sum avoided underflow or overflow.\"\n  #   pass\n\n  return result", "response": "Generates a log - sum of the log - values."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ntaking a log - values as input exponentiates them sums down the rows and returns the result.", "response": "def lscsum0(lx):\n  \"\"\"\n  Accepts log-values as input, exponentiates them, sums down the rows\n  (first dimension), then converts the sum back to log-space and returns the result.\n  Handles underflow by rescaling so that the largest values is exactly 1.0.\n  \"\"\"\n  # rows = lx.shape[0]\n  # columns = numpy.prod(lx.shape[1:])\n  # lx = lx.reshape(rows, columns)\n  # bases = lx.max(1).reshape(rows, 1)\n  # bases = lx.max(0).reshape((1,) + lx.shape[1:])\n  lx = numpy.asarray(lx)\n  bases = lx.max(0) # Don't need to reshape in the case of 0.\n  x = numpy.exp(lx - bases)\n  ssum = x.sum(0)\n\n  result = numpy.log(ssum) + bases\n  try:\n    conventional = numpy.log(numpy.exp(lx).sum(0))\n\n    if not similar(result, conventional):\n      if numpy.isinf(conventional).any() and not numpy.isinf(result).any():\n        # print \"Scaled log sum down axis 0 avoided underflow or overflow.\"\n        pass\n      else:\n        import sys\n        print >>sys.stderr, \"Warning: scaled log sum down axis 0 did not match.\"\n        print >>sys.stderr, \"Scaled log result:\"\n        print >>sys.stderr, result\n        print >>sys.stderr, \"Conventional result:\"\n        print >>sys.stderr, conventional\n  except FloatingPointError, e:\n    # print \"Scaled log sum down axis 0 avoided underflow or overflow.\"\n    pass\n\n\n  return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef normalize(lx):\n  lx = numpy.asarray(lx)\n  base = lx.max()\n  x = numpy.exp(lx - base)\n  result = x / x.sum()\n\n  conventional = (numpy.exp(lx) / numpy.exp(lx).sum())\n  assert similar(result, conventional)\n\n  return result", "response": "Normalizes the log - values in input and returns the result."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the sum of the values of the first dimension of the log - values.", "response": "def nsum0(lx):\n  \"\"\"\n  Accepts log-values as input, exponentiates them, sums down the rows\n  (first dimension), normalizes and returns the result.\n  Handles underflow by rescaling so that the largest values is exactly 1.0.\n  \"\"\"\n  lx = numpy.asarray(lx)\n  base = lx.max()\n  x = numpy.exp(lx - base)\n  ssum = x.sum(0)\n  result = ssum / ssum.sum()\n\n  conventional = (numpy.exp(lx).sum(0) / numpy.exp(lx).sum())\n  assert similar(result, conventional)\n\n  return result"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn log sum of exp of A and B", "response": "def logSumExp(A, B, out=None):\n  \"\"\" returns log(exp(A) + exp(B)). A and B are numpy arrays\"\"\"\n\n  if out is None:\n    out = numpy.zeros(A.shape)\n\n  indicator1 = A >= B\n  indicator2 = numpy.logical_not(indicator1)\n  out[indicator1] = A[indicator1] + numpy.log1p(numpy.exp(B[indicator1]-A[indicator1]))\n  out[indicator2]  = B[indicator2] + numpy.log1p(numpy.exp(A[indicator2]-B[indicator2]))\n\n  return out"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns log - exp - exp of A - B", "response": "def logDiffExp(A, B, out=None):\n  \"\"\" returns log(exp(A) - exp(B)). A and B are numpy arrays. values in A should be\n  greater than or equal to corresponding values in B\"\"\"\n\n  if out is None:\n    out = numpy.zeros(A.shape)\n\n  indicator1 = A >= B\n  assert indicator1.all(), \"Values in the first array should be greater than the values in the second\"\n  out[indicator1] = A[indicator1] + numpy.log(1 - numpy.exp(B[indicator1]-A[indicator1]))\n\n  return out"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _getFieldIndexBySpecial(fields, special):\n  for i, field in enumerate(fields):\n    if field.special == special:\n      return i\n  return None", "response": "Return the index of the field with the given meta special value."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef encode(self, inputRow):\n\n    # Create the return dict\n    result = dict(zip(self._fieldNames, inputRow))\n\n    # Add in the special fields\n    if self._categoryFieldIndex is not None:\n      # category value can be an int or a list\n      if isinstance(inputRow[self._categoryFieldIndex], int):\n        result['_category'] = [inputRow[self._categoryFieldIndex]]\n      else:\n        result['_category'] = (inputRow[self._categoryFieldIndex]\n                               if inputRow[self._categoryFieldIndex]\n                               else [None])\n    else:\n      result['_category'] = [None]\n\n    if self._resetFieldIndex is not None:\n      result['_reset'] = int(bool(inputRow[self._resetFieldIndex]))\n    else:\n      result['_reset'] = 0\n\n    if self._learningFieldIndex is not None:\n      result['_learning'] = int(bool(inputRow[self._learningFieldIndex]))\n\n    result['_timestampRecordIdx'] = None\n    if self._timestampFieldIndex is not None:\n      result['_timestamp'] = inputRow[self._timestampFieldIndex]\n      # Compute the record index based on timestamp\n      result['_timestampRecordIdx'] = self._computeTimestampRecordIdx(\n        inputRow[self._timestampFieldIndex])\n    else:\n      result['_timestamp'] = None\n\n    # -----------------------------------------------------------------------\n    # Figure out the sequence ID\n    hasReset = self._resetFieldIndex is not None\n    hasSequenceId = self._sequenceFieldIndex is not None\n    if hasReset and not hasSequenceId:\n      # Reset only\n      if result['_reset']:\n        self._sequenceId += 1\n      sequenceId = self._sequenceId\n\n    elif not hasReset and hasSequenceId:\n      sequenceId = inputRow[self._sequenceFieldIndex]\n      result['_reset'] = int(sequenceId != self._sequenceId)\n      self._sequenceId = sequenceId\n\n    elif hasReset and hasSequenceId:\n      sequenceId = inputRow[self._sequenceFieldIndex]\n\n    else:\n      sequenceId = 0\n\n    if sequenceId is not None:\n      result['_sequenceId'] = hash(sequenceId)\n    else:\n      result['_sequenceId'] = None\n\n    return result", "response": "Encodes the given input row as a dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _computeTimestampRecordIdx(self, recordTS):\n\n    if self._aggregationPeriod is None:\n      return None\n\n    # Base record index on number of elapsed months if aggregation is in\n    #  months\n    if self._aggregationPeriod['months'] > 0:\n      assert self._aggregationPeriod['seconds'] == 0\n      result = int(\n        (recordTS.year * 12 + (recordTS.month-1)) /\n        self._aggregationPeriod['months'])\n\n    # Base record index on elapsed seconds\n    elif self._aggregationPeriod['seconds'] > 0:\n      delta = recordTS - datetime.datetime(year=1, month=1, day=1)\n      deltaSecs = delta.days * 24 * 60 * 60   \\\n                + delta.seconds               \\\n                + delta.microseconds / 1000000.0\n      result = int(deltaSecs / self._aggregationPeriod['seconds'])\n\n    else:\n      result = None\n\n    return result", "response": "Given the timestamp of a record compute the record s timestamp index."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the next available data record as a dict.", "response": "def getNextRecordDict(self):\n    \"\"\"Returns next available data record from the storage as a dict, with the\n    keys being the field names. This also adds in some meta fields:\n\n      - ``_category``: The value from the category field (if any)\n      - ``_reset``: True if the reset field was True (if any)\n      - ``_sequenceId``: the value from the sequenceId field (if any)\n\n    \"\"\"\n\n    values = self.getNextRecord()\n    if values is None:\n      return None\n\n    if not values:\n      return dict()\n\n    if self._modelRecordEncoder is None:\n      self._modelRecordEncoder = ModelRecordEncoder(\n        fields=self.getFields(),\n        aggregationPeriod=self.getAggregationMonthsAndSeconds())\n\n    return self._modelRecordEncoder.encode(values)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the current minimum value for the given field name.", "response": "def getFieldMin(self, fieldName):\n    \"\"\"\n    If underlying implementation does not support min/max stats collection,\n    or if a field type does not support min/max (non scalars), the return\n    value will be None.\n\n    :param fieldName: (string) name of field to get min\n    :returns: current minimum value for the field ``fieldName``.\n    \"\"\"\n    stats = self.getStats()\n    if stats == None:\n      return None\n    minValues = stats.get('min', None)\n    if minValues == None:\n      return None\n    index = self.getFieldNames().index(fieldName)\n    return minValues[index]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the maximum value for the given field name.", "response": "def getFieldMax(self, fieldName):\n    \"\"\"\n    If underlying implementation does not support min/max stats collection,\n    or if a field type does not support min/max (non scalars), the return\n    value will be None.\n\n    :param fieldName: (string) name of field to get max\n    :returns: current maximum value for the field ``fieldName``.\n    \"\"\"\n    stats = self.getStats()\n    if stats == None:\n      return None\n    maxValues = stats.get('max', None)\n    if maxValues == None:\n      return None\n    index = self.getFieldNames().index(fieldName)\n    return maxValues[index]"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nlog msg % args with severity DEBUG.", "response": "def debug(self, msg, *args, **kwargs):\n    \"\"\"\n    Log 'msg % args' with severity 'DEBUG'.\n\n    To pass exception information, use the keyword argument exc_info with\n    a true value, e.g.\n\n    logger.debug(\"Houston, we have a %s\", \"thorny problem\", exc_info=1)\n    \"\"\"\n    self._baseLogger.debug(self, self.getExtendedMsg(msg), *args, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nlogging msg % args with severity INFO.", "response": "def info(self, msg, *args, **kwargs):\n    \"\"\"\n    Log 'msg % args' with severity 'INFO'.\n\n    To pass exception information, use the keyword argument exc_info with\n    a true value, e.g.\n\n    logger.info(\"Houston, we have a %s\", \"interesting problem\", exc_info=1)\n    \"\"\"\n    self._baseLogger.info(self, self.getExtendedMsg(msg), *args, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nlog msg % args with severity WARNING.", "response": "def warning(self, msg, *args, **kwargs):\n    \"\"\"\n    Log 'msg % args' with severity 'WARNING'.\n\n    To pass exception information, use the keyword argument exc_info with\n    a true value, e.g.\n\n    logger.warning(\"Houston, we have a %s\", \"bit of a problem\", exc_info=1)\n    \"\"\"\n    self._baseLogger.warning(self, self.getExtendedMsg(msg), *args, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nlogs msg % args with severity ERROR.", "response": "def error(self, msg, *args, **kwargs):\n    \"\"\"\n    Log 'msg % args' with severity 'ERROR'.\n\n    To pass exception information, use the keyword argument exc_info with\n    a true value, e.g.\n\n    logger.error(\"Houston, we have a %s\", \"major problem\", exc_info=1)\n    \"\"\"\n    self._baseLogger.error(self, self.getExtendedMsg(msg), *args, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef critical(self, msg, *args, **kwargs):\n    self._baseLogger.critical(self, self.getExtendedMsg(msg), *args, **kwargs)", "response": "Log msg % args with severity CRITICAL."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nlogging msg % args with the integer severity level.", "response": "def log(self, level, msg, *args, **kwargs):\n      \"\"\"\n      Log 'msg % args' with the integer severity 'level'.\n\n      To pass exception information, use the keyword argument exc_info with\n      a true value, e.g.\n\n      logger.log(level, \"We have a %s\", \"mysterious problem\", exc_info=1)\n      \"\"\"\n      self._baseLogger.log(self, level, self.getExtendedMsg(msg), *args,\n                           **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninitialize the internal filter variables for further processing.", "response": "def initFilter(input, filterInfo = None):\n  \"\"\" Initializes internal filter variables for further processing.\n  Returns a tuple (function to call,parameters for the filter call)\n\n  The filterInfo is a dict. Here is an example structure:\n    {fieldName: {'min': x,\n                 'max': y,\n                 'type': 'category', # or 'number'\n                 'acceptValues': ['foo', 'bar'],\n                 }\n    }\n\n  This returns the following:\n    (filterFunc, ((fieldIdx, fieldFilterFunc, filterDict),\n                  ...)\n\n  Where fieldIdx is the index of the field within each record\n        fieldFilterFunc returns True if the value is \"OK\" (within min, max or\n           part of acceptValues)\n        fieldDict is a dict containing 'type', 'min', max', 'acceptValues'\n  \"\"\"\n\n  if filterInfo is None:\n    return None\n\n  # Build an array of index/func to call on record[index]\n  filterList = []\n  for i, fieldName in enumerate(input.getFieldNames()):\n    fieldFilter = filterInfo.get(fieldName, None)\n    if fieldFilter == None:\n      continue\n\n    var = dict()\n    var['acceptValues'] = None\n    min = fieldFilter.get('min', None)\n    max = fieldFilter.get('max', None)\n    var['min'] = min\n    var['max'] = max\n\n\n    if fieldFilter['type'] == 'category':\n      var['acceptValues'] = fieldFilter['acceptValues']\n      fp = lambda x: (x['value'] != SENTINEL_VALUE_FOR_MISSING_DATA and \\\n                      x['value'] in x['acceptValues'])\n\n    elif fieldFilter['type'] == 'number':\n\n      if min != None and max != None:\n        fp = lambda x: (x['value'] != SENTINEL_VALUE_FOR_MISSING_DATA and \\\n                        x['value'] >= x['min'] and x['value'] <= x['max'])\n      elif min != None:\n        fp = lambda x: (x['value'] != SENTINEL_VALUE_FOR_MISSING_DATA and \\\n                        x['value'] >= x['min'])\n      else:\n        fp = lambda x: (x['value'] != SENTINEL_VALUE_FOR_MISSING_DATA and \\\n                        x['value'] <= x['max'])\n\n    filterList.append((i, fp, var))\n\n  return (_filterRecord, filterList)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _filterRecord(filterList, record):\n\n  for (fieldIdx, fp, params) in filterList:\n    x = dict()\n    x['value'] = record[fieldIdx]\n    x['acceptValues'] = params['acceptValues']\n    x['min'] = params['min']\n    x['max'] = params['max']\n    if not fp(x):\n      return False\n\n  # None of the field filters triggered, accept the record as a good one\n  return True", "response": "Takes a record and returns true if it meets the filter criteria and false otherwise."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _aggr_sum(inList):\n  aggrMean = _aggr_mean(inList)\n  if aggrMean == None:\n    return None\n\n  aggrSum = 0\n  for elem in inList:\n    if elem != SENTINEL_VALUE_FOR_MISSING_DATA:\n      aggrSum += elem\n    else:\n      aggrSum += aggrMean\n\n  return aggrSum", "response": "Returns the sum of the elements in the list. Missing items are replaced with\n athe."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _aggr_mean(inList):\n  aggrSum = 0\n  nonNone = 0\n  for elem in inList:\n    if elem != SENTINEL_VALUE_FOR_MISSING_DATA:\n      aggrSum += elem\n      nonNone += 1\n  if nonNone != 0:\n    return aggrSum / nonNone\n  else:\n    return None", "response": "Calculates the average of non - None elements of the list\n."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _aggr_mode(inList):\n\n  valueCounts = dict()\n  nonNone = 0\n\n  for elem in inList:\n    if elem == SENTINEL_VALUE_FOR_MISSING_DATA:\n      continue\n\n    nonNone += 1\n    if elem in valueCounts:\n      valueCounts[elem] += 1\n    else:\n      valueCounts[elem] = 1\n\n  # Get the most common one\n  if nonNone == 0:\n    return None\n\n  # Sort by counts\n  sortedCounts = valueCounts.items()\n  sortedCounts.sort(cmp=lambda x,y: x[1] - y[1], reverse=True)\n  return sortedCounts[0][0]", "response": "Returns the most common value seen in the non - None elements of the list\n \"\"\""}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _aggr_weighted_mean(inList, params):\n  assert(len(inList) == len(params))\n\n  # If all weights are 0, then the value is not defined, return None (missing)\n  weightsSum = sum(params)\n  if weightsSum == 0:\n    return None\n\n  weightedMean = 0\n  for i, elem in enumerate(inList):\n    weightedMean += elem * params[i]\n\n  return weightedMean / weightsSum", "response": "Calculates the weighted mean of inList based on the given parameters."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef generateDataset(aggregationInfo, inputFilename, outputFilename=None):\n\n\n\n  # Create the input stream\n  inputFullPath = resource_filename(\"nupic.datafiles\", inputFilename)\n  inputObj = FileRecordStream(inputFullPath)\n\n\n  # Instantiate the aggregator\n  aggregator = Aggregator(aggregationInfo=aggregationInfo,\n                          inputFields=inputObj.getFields())\n\n\n  # Is it a null aggregation? If so, just return the input file unmodified\n  if aggregator.isNullAggregation():\n    return inputFullPath\n\n\n  # ------------------------------------------------------------------------\n  # If we were not given an output filename, create one based on the\n  #  aggregation settings\n  if outputFilename is None:\n    outputFilename = 'agg_%s' % \\\n                        os.path.splitext(os.path.basename(inputFullPath))[0]\n    timePeriods = 'years months weeks days '\\\n                  'hours minutes seconds milliseconds microseconds'\n    for k in timePeriods.split():\n      if aggregationInfo.get(k, 0) > 0:\n        outputFilename += '_%s_%d' % (k, aggregationInfo[k])\n\n    outputFilename += '.csv'\n    outputFilename = os.path.join(os.path.dirname(inputFullPath), outputFilename)\n\n\n\n  # ------------------------------------------------------------------------\n  # If some other process already started creating this file, simply\n  #   wait for it to finish and return without doing anything\n  lockFilePath = outputFilename + '.please_wait'\n  if os.path.isfile(outputFilename) or \\\n     os.path.isfile(lockFilePath):\n    while os.path.isfile(lockFilePath):\n      print 'Waiting for %s to be fully written by another process' % \\\n            lockFilePath\n      time.sleep(1)\n    return outputFilename\n\n\n  # Create the lock file\n  lockFD = open(lockFilePath, 'w')\n\n\n\n  # -------------------------------------------------------------------------\n  # Create the output stream\n  outputObj = FileRecordStream(streamID=outputFilename, write=True,\n                               fields=inputObj.getFields())\n\n\n  # -------------------------------------------------------------------------\n  # Write all aggregated records to the output\n  while True:\n    inRecord = inputObj.getNextRecord()\n\n    (aggRecord, aggBookmark) = aggregator.next(inRecord, None)\n\n    if aggRecord is None and inRecord is None:\n      break\n\n    if aggRecord is not None:\n      outputObj.appendRecord(aggRecord)\n\n  return outputFilename", "response": "Generates a dataset of aggregated values from the given file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating the filename for aggregated dataset .", "response": "def getFilename(aggregationInfo, inputFile):\n  \"\"\"Generate the filename for aggregated dataset\n\n  The filename is based on the input filename and the\n  aggregation period.\n\n  Returns the inputFile if no aggregation required (aggregation\n  info has all 0's)\n  \"\"\"\n\n  # Find the actual file, with an absolute path\n  inputFile = resource_filename(\"nupic.datafiles\", inputFile)\n\n  a = defaultdict(lambda: 0, aggregationInfo)\n  outputDir = os.path.dirname(inputFile)\n  outputFile = 'agg_%s' % os.path.splitext(os.path.basename(inputFile))[0]\n  noAggregation = True\n  timePeriods = 'years months weeks days '\\\n                'hours minutes seconds milliseconds microseconds'\n  for k in timePeriods.split():\n    if a[k] > 0:\n      noAggregation = False\n      outputFile += '_%s_%d' % (k, a[k])\n\n  if noAggregation:\n    return inputFile\n  outputFile += '.csv'\n  outputFile = os.path.join(outputDir, outputFile)\n\n  return outputFile"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _getEndTime(self, t):\n\n    assert isinstance(t, datetime.datetime)\n    if self._aggTimeDelta:\n      return t + self._aggTimeDelta\n    else:\n      year = t.year + self._aggYears + (t.month - 1 + self._aggMonths) / 12\n      month = (t.month - 1 + self._aggMonths) % 12 + 1\n      return t.replace(year=year, month=month)", "response": "Add the aggregation period to the input time t and return a datetime object."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _getFuncPtrAndParams(self, funcName):\n\n    params = None\n    if isinstance(funcName, basestring):\n      if funcName == 'sum':\n        fp = _aggr_sum\n      elif funcName == 'first':\n        fp = _aggr_first\n      elif funcName == 'last':\n        fp = _aggr_last\n      elif funcName == 'mean':\n        fp = _aggr_mean\n      elif funcName == 'max':\n        fp = max\n      elif funcName == 'min':\n        fp = min\n      elif funcName == 'mode':\n        fp = _aggr_mode\n      elif funcName.startswith('wmean:'):\n        fp = _aggr_weighted_mean\n        paramsName = funcName[6:]\n        params = [f[0] for f in self._inputFields].index(paramsName)\n    else:\n      fp = funcName\n\n    return (fp, params)", "response": "Returns the function pointer and param for the given aggregation function."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _createAggregateRecord(self):\n\n    record = []\n\n    for i, (fieldIdx, aggFP, paramIdx) in enumerate(self._fields):\n      if aggFP is None: # this field is not supposed to be aggregated.\n        continue\n\n      values = self._slice[i]\n      refIndex = None\n      if paramIdx is not None:\n        record.append(aggFP(values, self._slice[paramIdx]))\n      else:\n        record.append(aggFP(values))\n\n    return record", "response": "Generates the aggregated output record for this object."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef next(self, record, curInputBookmark):\n\n    # This will hold the aggregated record we return\n    outRecord = None\n\n    # This will hold the bookmark of the last input used within the\n    #  aggregated record we return.\n    retInputBookmark = None\n\n    if record is not None:\n\n      # Increment input count\n      self._inIdx += 1\n\n      #print self._inIdx, record\n\n      # Apply the filter, ignore the record if any field is unacceptable\n      if self._filter != None and not self._filter[0](self._filter[1], record):\n        return (None, None)\n\n      # If no aggregation info just return as-is\n      if self._nullAggregation:\n        return (record, curInputBookmark)\n\n\n      # ----------------------------------------------------------------------\n      # Do aggregation\n\n      #\n      # Remember the very first record time stamp - it will be used as\n      # the timestamp for all first records in all sequences to align\n      # times for the aggregation/join of sequences.\n      #\n      # For a set of aggregated records, it will use the beginning of the time\n      # window as a timestamp for the set\n      #\n      t = record[self._timeFieldIdx]\n\n      if self._firstSequenceStartTime == None:\n        self._firstSequenceStartTime = t\n\n      # Create initial startTime and endTime if needed\n      if self._startTime is None:\n        self._startTime = t\n      if self._endTime is None:\n        self._endTime = self._getEndTime(t)\n        assert self._endTime > t\n\n      #print 'Processing line:', i, t, endTime\n      #from dbgp.client import brk; brk(port=9011)\n\n\n      # ----------------------------------------------------------------------\n      # Does this record have a reset signal or sequence Id associated with it?\n      # If so, see if we've reached a sequence boundary\n      if self._resetFieldIdx is not None:\n        resetSignal = record[self._resetFieldIdx]\n      else:\n        resetSignal = None\n\n      if self._sequenceIdFieldIdx is not None:\n        currSequenceId = record[self._sequenceIdFieldIdx]\n      else:\n        currSequenceId = None\n\n      newSequence = (resetSignal == 1 and self._inIdx > 0) \\\n                      or self._sequenceId != currSequenceId \\\n                      or self._inIdx == 0\n\n      if newSequence:\n        self._sequenceId = currSequenceId\n\n\n      # --------------------------------------------------------------------\n      # We end the aggregation chunk if we go past the end time\n      # -OR- we get an out of order record (t < startTime)\n      sliceEnded = (t >= self._endTime or t < self._startTime)\n\n\n      # -------------------------------------------------------------------\n      # Time to generate a new output record?\n      if (newSequence or sliceEnded) and len(self._slice) > 0:\n        # Create aggregated record\n        # print 'Creating aggregate record...'\n\n        # Make first record timestamp as the beginning of the time period,\n        # in case the first record wasn't falling on the beginning of the period\n        for j, f in enumerate(self._fields):\n          index = f[0]\n          if index == self._timeFieldIdx:\n            self._slice[j][0] = self._startTime\n            break\n\n        # Generate the aggregated record\n        outRecord = self._createAggregateRecord()\n        retInputBookmark = self._aggrInputBookmark\n\n        # Reset the slice\n        self._slice = defaultdict(list)\n\n\n      # --------------------------------------------------------------------\n      # Add current record to slice (Note keeping slices in memory). Each\n      # field in the slice is a list of field values from all the sliced\n      # records\n      for j, f in enumerate(self._fields):\n        index = f[0]\n        # append the parsed field value to the proper aggregated slice field.\n        self._slice[j].append(record[index])\n        self._aggrInputBookmark = curInputBookmark\n\n\n      # --------------------------------------------------------------------\n      # If we've encountered a new sequence, start aggregation over again\n      if newSequence:\n        # TODO: May use self._firstSequenceStartTime as a start for the new\n        # sequence (to align all sequences)\n        self._startTime = t\n        self._endTime = self._getEndTime(t)\n\n\n      # --------------------------------------------------------------------\n      # If a slice just ended, re-compute the start and end time for the\n      #  next aggregated record\n      if sliceEnded:\n        # Did we receive an out of order record? If so, go back and iterate\n        #   till we get to the next end time boundary.\n        if t < self._startTime:\n          self._endTime = self._firstSequenceStartTime\n        while t >= self._endTime:\n          self._startTime = self._endTime\n          self._endTime = self._getEndTime(self._endTime)\n\n\n      # If we have a record to return, do it now\n      if outRecord is not None:\n        return (outRecord, retInputBookmark)\n\n\n    # ---------------------------------------------------------------------\n    # Input reached EOF\n    # Aggregate one last time in the end if necessary\n    elif self._slice:\n\n      # Make first record timestamp as the beginning of the time period,\n      # in case the first record wasn't falling on the beginning of the period\n      for j, f in enumerate(self._fields):\n        index = f[0]\n        if index == self._timeFieldIdx:\n          self._slice[j][0] = self._startTime\n          break\n\n      outRecord = self._createAggregateRecord()\n      retInputBookmark = self._aggrInputBookmark\n\n      self._slice = defaultdict(list)\n\n\n    # Return aggregated record\n    return (outRecord, retInputBookmark)", "response": "Returns the next aggregated record in the sequence of records."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef processClubAttendance(f, clubs):\n  try:\n    # Skip as many empty lines as necessary (file format inconsistent)\n    line = f.next()\n    while line == ',,,,,,,,,,,,,,,,,,,\\n':\n      line = f.next()\n    \n    # The first non-empty line should have the name as the first field\n    name = line.split(',')[0]\n    \n    # Create a new club object if needed\n    if name not in clubs:\n      clubs[name] = Club(name)\n    \n    # Get the named club\n    c = clubs[name]\n    \n    c.processAttendance(f)      \n    return True\n  except StopIteration:\n    return False", "response": "Process the attendance data of one club\n  \n"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef processClubConsumption(f, clubs):\n  try:\n    # Skip header line\n    line = f.next()\n    assert line.endswith('\"   \",\"SITE_LOCATION_NAME\",\"TIMESTAMP\",\"TOTAL_KWH\"\\n')\n\n    valid_times = range(24)\n    t = 0 # used to track time\n    club = None\n    clubName = None\n    lastDate = None\n    while True:\n      assert t in valid_times\n      consumption = 0\n      for x in range(4):\n        # Read the line and get rid of the newline character\n        line = f.next()[:-1]\n        fields = line.split(',')\n        assert len(fields) == 4\n        for i, field in enumerate(fields):\n          # Strip the redundant double quotes\n          assert field[0] == '\"' and field[-1] == '\"'\n          fields[i] = field[1:-1]\n        \n        # Ignoring field 0, which is just a running count\n        \n        # Get the club name  \n        name = fields[1]\n        \n        # Hack to fix inconsistent club names like: \"Melbourne CBD - Melbourne Central\" vs. \"Melbourne Central\"\n        partialNames = ('Melbourne Central', 'North Sydney', 'Park St', 'Pitt St')\n        for pn in partialNames:\n          if pn in name:\n            name = pn\n        \n        # Locate the club if needed (maybe )\n        if name != clubName:\n          clubName = name\n          club = clubs[name]\n        \n        # Split the date (time is counted using the t variable)\n        tokens = fields[2].split()\n        \n        # Verify that t == 0 and consumption == 0 when there is no time in the file\n        if len(tokens) == 1:\n          assert consumption == 0 and t == 0\n        \n        # The first (and sometimes only) token is the date\n        date = tokens[0]\n                \n        # Aggregate the consumption\n        consumption += float(fields[3])\n      \n      # Update the Club object after aggregating the consumption of 4 lines \n      club.updateRecord(date, t, consumption)\n      \n      # Increment time\n      t += 1\n      t %= 24\n  except StopIteration:\n    return", "response": "Process the consumption a clubConsumption a clubVersion is a bit more efficient than reading the consumption a clubVersion is a bit more efficient than reading the clubVersion and parsing the line until the next line."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrun one iteration of this model.", "response": "def run(self, inputRecord):\n    \"\"\"\n    Run one iteration of this model.\n\n    :param inputRecord: (object)\n           A record object formatted according to\n           :meth:`~nupic.data.record_stream.RecordStreamIface.getNextRecord` or\n           :meth:`~nupic.data.record_stream.RecordStreamIface.getNextRecordDict`\n           result format.\n    :returns: (:class:`~nupic.frameworks.opf.opf_utils.ModelResult`)\n             An ModelResult namedtuple. The contents of ModelResult.inferences\n             depends on the the specific inference type of this model, which\n             can be queried by :meth:`.getInferenceType`.\n    \"\"\"\n    # 0-based prediction index for ModelResult\n    predictionNumber = self._numPredictions\n    self._numPredictions += 1\n    result = opf_utils.ModelResult(predictionNumber=predictionNumber,\n                                   rawInput=inputRecord)\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _getModelCheckpointFilePath(checkpointDir):\n    path = os.path.join(checkpointDir, \"model.data\")\n    path = os.path.abspath(path)\n    return path", "response": "Returns the absolute path of the model s checkpoint file."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nserialize model using capnproto and writes data to checkpointDir", "response": "def writeToCheckpoint(self, checkpointDir):\n    \"\"\"Serializes model using capnproto and writes data to ``checkpointDir``\"\"\"\n    proto = self.getSchema().new_message()\n\n    self.write(proto)\n\n    checkpointPath = self._getModelCheckpointFilePath(checkpointDir)\n\n    # Clean up old saved state, if any\n    if os.path.exists(checkpointDir):\n      if not os.path.isdir(checkpointDir):\n        raise Exception((\"Existing filesystem entry <%s> is not a model\"\n                         \" checkpoint -- refusing to delete (not a directory)\") \\\n                          % checkpointDir)\n      if not os.path.isfile(checkpointPath):\n        raise Exception((\"Existing filesystem entry <%s> is not a model\"\n                         \" checkpoint -- refusing to delete\"\\\n                         \" (%s missing or not a file)\") % \\\n                          (checkpointDir, checkpointPath))\n\n      shutil.rmtree(checkpointDir)\n\n    # Create a new directory for saving state\n    self.__makeDirectoryFromAbsolutePath(checkpointDir)\n\n    with open(checkpointPath, 'wb') as f:\n      proto.write(f)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef readFromCheckpoint(cls, checkpointDir):\n    checkpointPath = cls._getModelCheckpointFilePath(checkpointDir)\n\n    with open(checkpointPath, 'r') as f:\n      proto = cls.getSchema().read(f,\n                                   traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)\n\n    model = cls.read(proto)\n    return model", "response": "Deserializes model from checkpointDir using capnproto"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef writeBaseToProto(self, proto):\n    inferenceType = self.getInferenceType()\n    # lower-case first letter to be compatible with capnproto enum naming\n    inferenceType = inferenceType[:1].lower() + inferenceType[1:]\n    proto.inferenceType = inferenceType\n\n    proto.numPredictions = self._numPredictions\n\n    proto.learningEnabled = self.__learningEnabled\n    proto.inferenceEnabled = self.__inferenceEnabled\n    proto.inferenceArgs = json.dumps(self.__inferenceArgs)", "response": "Save the state maintained by the Model base class to the given capnp. ModelProto message builder."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef save(self, saveModelDir):\n    logger = self._getLogger()\n    logger.debug(\"(%s) Creating local checkpoint in %r...\",\n                       self, saveModelDir)\n\n    modelPickleFilePath = self._getModelPickleFilePath(saveModelDir)\n\n    # Clean up old saved state, if any\n    if os.path.exists(saveModelDir):\n      if not os.path.isdir(saveModelDir):\n        raise Exception((\"Existing filesystem entry <%s> is not a model\"\n                         \" checkpoint -- refusing to delete (not a directory)\") \\\n                          % saveModelDir)\n      if not os.path.isfile(modelPickleFilePath):\n        raise Exception((\"Existing filesystem entry <%s> is not a model\"\n                         \" checkpoint -- refusing to delete\"\\\n                         \" (%s missing or not a file)\") % \\\n                          (saveModelDir, modelPickleFilePath))\n\n      shutil.rmtree(saveModelDir)\n\n    # Create a new directory for saving state\n    self.__makeDirectoryFromAbsolutePath(saveModelDir)\n\n    with open(modelPickleFilePath, 'wb') as modelPickleFile:\n      logger.debug(\"(%s) Pickling Model instance...\", self)\n\n      pickle.dump(self, modelPickleFile, protocol=pickle.HIGHEST_PROTOCOL)\n\n      logger.debug(\"(%s) Finished pickling Model instance\", self)\n\n\n    # Tell the model to save extra data, if any, that's too big for pickling\n    self._serializeExtraData(extraDataDir=self._getModelExtraDataDir(saveModelDir))\n\n    logger.debug(\"(%s) Finished creating local checkpoint\", self)\n\n    return", "response": "Save the current state of the current model in the given directory."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef load(cls, savedModelDir):\n    logger = opf_utils.initLogger(cls)\n    logger.debug(\"Loading model from local checkpoint at %r...\", savedModelDir)\n\n    # Load the model\n    modelPickleFilePath = Model._getModelPickleFilePath(savedModelDir)\n\n    with open(modelPickleFilePath, 'rb') as modelPickleFile:\n      logger.debug(\"Unpickling Model instance...\")\n\n      model = pickle.load(modelPickleFile)\n\n      logger.debug(\"Finished unpickling Model instance\")\n\n    # Tell the model to load extra data, if any, that was too big for pickling\n    model._deSerializeExtraData(\n        extraDataDir=Model._getModelExtraDataDir(savedModelDir))\n\n    logger.debug(\"Finished Loading model from local checkpoint\")\n\n    return model", "response": "Loads the saved model."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the absolute path of the model s pickle file.", "response": "def _getModelPickleFilePath(saveModelDir):\n    \"\"\" Return the absolute path of the model's pickle file.\n\n    :param saveModelDir: (string)\n           Directory of where the experiment is to be or was saved\n    :returns: (string) An absolute path.\n    \"\"\"\n    path = os.path.join(saveModelDir, \"model.pkl\")\n    path = os.path.abspath(path)\n    return path"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _getModelExtraDataDir(saveModelDir):\n    path = os.path.join(saveModelDir, \"modelextradata\")\n    path = os.path.abspath(path)\n\n    return path", "response": "Returns the absolute path to the directory where the model s own\n    extra data are stored."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nrun an experiment on the given list of arguments and returns the model instance that was created.", "response": "def runExperiment(args, model=None):\n  \"\"\"\n  Run a single OPF experiment.\n\n  .. note:: The caller is responsible for initializing python logging before\n     calling this function (e.g., import :mod:`nupic.support`;\n     :meth:`nupic.support.initLogging`)\n\n  See also: :meth:`.initExperimentPrng`.\n\n  :param args: (string) Experiment command-line args list. Too see all options,\n      run with ``--help``:\n\n      .. code-block:: text\n\n        Options:\n          -h, --help           show this help message and exit\n          -c <CHECKPOINT>      Create a model and save it under the given <CHECKPOINT>\n                               name, but don't run it\n          --listCheckpoints    List all available checkpoints\n          --listTasks          List all task labels in description.py\n          --load=<CHECKPOINT>  Load a model from the given <CHECKPOINT> and run it.\n                               Run with --listCheckpoints flag for more details.\n          --newSerialization   Use new capnproto serialization\n          --tasks              Run the tasks with the given TASK LABELS in the order\n                               they are given.  Either end of arg-list, or a\n                               standalone dot ('.') arg or the next short or long\n                               option name (-a or --blah) terminates the list. NOTE:\n                               FAILS TO RECOGNIZE task label names with one or more\n                               leading dashes. [default: run all of the tasks in\n                               description.py]\n          --testMode           Reduce iteration count for testing\n          --noCheckpoint       Don't checkpoint the model after running each task.\n\n  :param model: (:class:`~nupic.frameworks.opf.model.Model`) For testing, may\n      pass in an existing OPF Model to use instead of creating a new one.\n\n  :returns: (:class:`~nupic.frameworks.opf.model.Model`)\n    reference to OPF Model instance that was constructed (this\n    is provided to aid with debugging) or None, if none was\n    created.\n  \"\"\"\n  # Parse command-line options\n  opt = _parseCommandLineOptions(args)\n\n  #print \"runExperiment: Parsed Command Options: \", opt\n\n  model = _runExperimentImpl(opt, model)\n\n  return model"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _parseCommandLineOptions(args):\n  usageStr = (\n    \"%prog [options] descriptionPyDirectory\\n\"\n    \"This script runs a single OPF Model described by description.py \"\n    \"located in the given directory.\"\n    )\n\n  parser = optparse.OptionParser(usage=usageStr)\n\n  parser.add_option(\"-c\",\n                    help=\"Create a model and save it under the given \"\n                         \"<CHECKPOINT> name, but don't run it\",\n                    dest=\"createCheckpointName\",\n                    action=\"store\", type=\"string\", default=\"\",\n                    metavar=\"<CHECKPOINT>\")\n\n  parser.add_option(\"--listCheckpoints\",\n                    help=\"List all available checkpoints\",\n                    dest=\"listAvailableCheckpoints\",\n                    action=\"store_true\", default=False)\n\n  parser.add_option(\"--listTasks\",\n                    help=\"List all task labels in description.py\",\n                    dest=\"listTasks\",\n                    action=\"store_true\", default=False)\n\n  parser.add_option(\"--load\",\n                    help=\"Load a model from the given <CHECKPOINT> and run it. \"\n                         \"Run with --listCheckpoints flag for more details. \",\n                    dest=\"runCheckpointName\",\n                    action=\"store\", type=\"string\", default=\"\",\n                    metavar=\"<CHECKPOINT>\")\n\n  parser.add_option(\"--newSerialization\",\n                    help=\"Use new capnproto serialization\",\n                    dest=\"newSerialization\",\n                    action=\"store_true\", default=False)\n\n  #parser.add_option(\"--reuseDatasets\",\n  #                  help=\"Keep existing generated/aggregated datasets\",\n  #                  dest=\"reuseDatasets\", action=\"store_true\",\n  #                  default=False)\n\n  parser.add_option(\"--tasks\",\n                    help=\"Run the tasks with the given TASK LABELS \"\n                         \"in the order they are given.  Either end of \"\n                         \"arg-list, or a standalone dot ('.') arg or \"\n                         \"the next short or long option name (-a or \"\n                         \"--blah) terminates the list. NOTE: FAILS \"\n                         \"TO RECOGNIZE task label names with one or more \"\n                         \"leading dashes. [default: run all of the tasks in \"\n                         \"description.py]\",\n                    dest=\"taskLabels\", default=[],\n                    action=\"callback\", callback=reapVarArgsCallback,\n                    metavar=\"TASK_LABELS\")\n\n  parser.add_option(\"--testMode\",\n                    help=\"Reduce iteration count for testing\",\n                    dest=\"testMode\", action=\"store_true\",\n                    default=False)\n\n  parser.add_option(\"--noCheckpoint\",\n                    help=\"Don't checkpoint the model after running each task.\",\n                    dest=\"checkpointModel\", action=\"store_false\",\n                    default=True)\n\n  options, experiments = parser.parse_args(args)\n\n  # Validate args\n  mutuallyExclusiveOptionCount = sum([bool(options.createCheckpointName),\n                                      options.listAvailableCheckpoints,\n                                      options.listTasks,\n                                      bool(options.runCheckpointName)])\n  if mutuallyExclusiveOptionCount > 1:\n    _reportCommandLineUsageErrorAndExit(\n        parser,\n        \"Options: -c, --listCheckpoints, --listTasks, and --load are \"\n        \"mutually exclusive. Please select only one\")\n\n  mutuallyExclusiveOptionCount = sum([bool(not options.checkpointModel),\n                                      bool(options.createCheckpointName)])\n  if mutuallyExclusiveOptionCount > 1:\n    _reportCommandLineUsageErrorAndExit(\n        parser,\n        \"Options: -c and --noCheckpoint are \"\n        \"mutually exclusive. Please select only one\")\n\n  if len(experiments) != 1:\n    _reportCommandLineUsageErrorAndExit(\n        parser,\n        \"Exactly ONE experiment must be specified, but got %s (%s)\" % (\n            len(experiments), experiments))\n\n  # Done with parser\n  parser.destroy()\n\n  # Prepare results\n\n  # Directory path of the experiment (that contain description.py)\n  experimentDir = os.path.abspath(experiments[0])\n\n  # RunExperiment.py's private options (g_parsedPrivateCommandLineOptionsSchema)\n  privateOptions = dict()\n  privateOptions['createCheckpointName'] = options.createCheckpointName\n  privateOptions['listAvailableCheckpoints'] = options.listAvailableCheckpoints\n  privateOptions['listTasks'] = options.listTasks\n  privateOptions['runCheckpointName'] = options.runCheckpointName\n  privateOptions['newSerialization'] = options.newSerialization\n  privateOptions['testMode'] = options.testMode\n  #privateOptions['reuseDatasets']  = options.reuseDatasets\n  privateOptions['taskLabels'] = options.taskLabels\n  privateOptions['checkpointModel'] = options.checkpointModel\n\n  result = ParseCommandLineOptionsResult(experimentDir=experimentDir,\n                                         privateOptions=privateOptions)\n  return result", "response": "Parse command line options and return a namedtuple with the parsed options."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef reapVarArgsCallback(option, optStr, value, parser):\n  newValues = []\n\n  # Reap the args, taking care to stop before the next option or '.'\n  gotDot = False\n  for arg in parser.rargs:\n    # Stop on --longname options\n    if arg.startswith(\"--\") and len(arg) > 2:\n      break\n\n    # Stop on -b options\n    if arg.startswith(\"-\") and len(arg) > 1:\n      break\n\n    if arg == \".\":\n      gotDot = True\n      break\n\n    newValues.append(arg)\n\n  if not newValues:\n    raise optparse.OptionValueError(\n      (\"Empty arg list for option %r expecting one or more args \"\n       \"(remaining tokens: %r)\") % (optStr, parser.rargs))\n\n  del parser.rargs[:len(newValues) + int(gotDot)]\n\n  # Retrieve the existing arg accumulator, if any\n  value = getattr(parser.values, option.dest, [])\n  #print \"Previous value: %r\" % value\n  if value is None:\n    value = []\n\n  # Append the new args to the existing ones and save to the parser\n  value.extend(newValues)\n  setattr(parser.values, option.dest, value)", "response": "This callback is used to reap the variable number of option args. It will reap the args and save the value to the parser. values attribute."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreporting usage error and exit program with error indication.", "response": "def _reportCommandLineUsageErrorAndExit(parser, message):\n  \"\"\"Report usage error and exit program with error indication.\"\"\"\n  print parser.get_usage()\n  print message\n  sys.exit(1)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate and runs the experiment from the given options and model.", "response": "def _runExperimentImpl(options, model=None):\n  \"\"\"Creates and runs the experiment\n\n  Args:\n    options: namedtuple ParseCommandLineOptionsResult\n    model: For testing: may pass in an existing OPF Model instance\n        to use instead of creating a new one.\n\n  Returns: reference to OPFExperiment instance that was constructed (this\n      is provided to aid with debugging) or None, if none was\n      created.\n  \"\"\"\n  json_helpers.validate(options.privateOptions,\n                        schemaDict=g_parsedPrivateCommandLineOptionsSchema)\n\n  # Load the experiment's description.py module\n  experimentDir = options.experimentDir\n  descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir(\n      experimentDir)\n  expIface = helpers.getExperimentDescriptionInterfaceFromModule(\n      descriptionPyModule)\n\n  # Handle \"list checkpoints\" request\n  if options.privateOptions['listAvailableCheckpoints']:\n    _printAvailableCheckpoints(experimentDir)\n    return None\n\n  # Load experiment tasks\n  experimentTasks = expIface.getModelControl().get('tasks', [])\n\n  # If the tasks list is empty, and this is a nupic environment description\n  # file being run from the OPF, convert it to a simple OPF description file.\n  if (len(experimentTasks) == 0 and\n      expIface.getModelControl()['environment'] == OpfEnvironment.Nupic):\n    expIface.convertNupicEnvToOPF()\n    experimentTasks = expIface.getModelControl().get('tasks', [])\n\n  # Ensures all the source locations are either absolute paths or relative to\n  # the nupic.datafiles package_data location.\n  expIface.normalizeStreamSources()\n\n  # Extract option\n  newSerialization = options.privateOptions['newSerialization']\n\n  # Handle listTasks\n  if options.privateOptions['listTasks']:\n    print \"Available tasks:\"\n\n    for label in [t['taskLabel'] for t in experimentTasks]:\n      print \"\\t\", label\n\n    return None\n\n  # Construct the experiment instance\n  if options.privateOptions['runCheckpointName']:\n\n    assert model is None\n\n    checkpointName = options.privateOptions['runCheckpointName']\n\n    model = ModelFactory.loadFromCheckpoint(\n          savedModelDir=_getModelCheckpointDir(experimentDir, checkpointName),\n          newSerialization=newSerialization)\n\n  elif model is not None:\n    print \"Skipping creation of OPFExperiment instance: caller provided his own\"\n  else:\n    modelDescription = expIface.getModelDescription()\n    model = ModelFactory.create(modelDescription)\n\n  # Handle \"create model\" request\n  if options.privateOptions['createCheckpointName']:\n    checkpointName = options.privateOptions['createCheckpointName']\n    _saveModel(model=model,\n               experimentDir=experimentDir,\n               checkpointLabel=checkpointName,\n               newSerialization=newSerialization)\n\n    return model\n\n  # Build the task list\n\n  # Default task execution index list is in the natural list order of the tasks\n  taskIndexList = range(len(experimentTasks))\n\n  customTaskExecutionLabelsList = options.privateOptions['taskLabels']\n  if customTaskExecutionLabelsList:\n    taskLabelsList = [t['taskLabel'] for t in experimentTasks]\n    taskLabelsSet = set(taskLabelsList)\n\n    customTaskExecutionLabelsSet = set(customTaskExecutionLabelsList)\n\n    assert customTaskExecutionLabelsSet.issubset(taskLabelsSet), \\\n           (\"Some custom-provided task execution labels don't correspond \"\n            \"to actual task labels: mismatched labels: %r; actual task \"\n            \"labels: %r.\") % (customTaskExecutionLabelsSet - taskLabelsSet,\n                              customTaskExecutionLabelsList)\n\n    taskIndexList = [taskLabelsList.index(label) for label in\n                     customTaskExecutionLabelsList]\n\n    print \"#### Executing custom task list: %r\" % [taskLabelsList[i] for\n                                                   i in taskIndexList]\n\n  # Run all experiment tasks\n  for taskIndex in taskIndexList:\n\n    task = experimentTasks[taskIndex]\n\n    # Create a task runner and run it!\n    taskRunner = _TaskRunner(model=model,\n                             task=task,\n                             cmdOptions=options)\n    taskRunner.run()\n    del taskRunner\n\n    if options.privateOptions['checkpointModel']:\n      _saveModel(model=model,\n                 experimentDir=experimentDir,\n                 checkpointLabel=task['taskLabel'],\n                 newSerialization=newSerialization)\n\n  return model"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _saveModel(model, experimentDir, checkpointLabel, newSerialization=False):\n  checkpointDir = _getModelCheckpointDir(experimentDir, checkpointLabel)\n  if newSerialization:\n    model.writeToCheckpoint(checkpointDir)\n  else:\n    model.save(saveModelDir=checkpointDir)", "response": "Save a model to checkpoint."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _getModelCheckpointDir(experimentDir, checkpointLabel):\n  checkpointDir = os.path.join(getCheckpointParentDir(experimentDir),\n                               checkpointLabel + g_defaultCheckpointExtension)\n  checkpointDir = os.path.abspath(checkpointDir)\n\n  return checkpointDir", "response": "Creates the directory for serialization of the model\n"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getCheckpointParentDir(experimentDir):\n  baseDir = os.path.join(experimentDir, \"savedmodels\")\n  baseDir = os.path.abspath(baseDir)\n\n  return baseDir", "response": "Get the absolute path to the base serialization directory within which\n      model checkpoints are created"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _checkpointLabelFromCheckpointDir(checkpointDir):\n  assert checkpointDir.endswith(g_defaultCheckpointExtension)\n\n  lastSegment = os.path.split(checkpointDir)[1]\n\n  checkpointLabel = lastSegment[0:-len(g_defaultCheckpointExtension)]\n\n  return checkpointLabel", "response": "Returns a checkpoint label string for the given checkpoint directory."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning true iff checkpointDir appears to be a checkpoint directory.", "response": "def _isCheckpointDir(checkpointDir):\n  \"\"\"Return true iff checkpointDir appears to be a checkpoint directory.\"\"\"\n  lastSegment = os.path.split(checkpointDir)[1]\n  if lastSegment[0] == '.':\n    return False\n\n  if not checkpointDir.endswith(g_defaultCheckpointExtension):\n    return False\n\n  if not os.path.isdir(checkpointDir):\n    return False\n\n  return True"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _printAvailableCheckpoints(experimentDir):\n  checkpointParentDir = getCheckpointParentDir(experimentDir)\n\n  if not os.path.exists(checkpointParentDir):\n    print \"No available checkpoints.\"\n    return\n\n  checkpointDirs = [x for x in os.listdir(checkpointParentDir)\n                    if _isCheckpointDir(os.path.join(checkpointParentDir, x))]\n  if not checkpointDirs:\n    print \"No available checkpoints.\"\n    return\n\n  print \"Available checkpoints:\"\n  checkpointList = [_checkpointLabelFromCheckpointDir(x)\n                    for x in checkpointDirs]\n\n  for checkpoint in sorted(checkpointList):\n    print \"\\t\", checkpoint\n\n  print\n  print \"To start from a checkpoint:\"\n  print \"  python run_opf_experiment.py experiment --load <CHECKPOINT>\"\n  print \"For example, to start from the checkpoint \\\"MyCheckpoint\\\":\"\n  print \"  python run_opf_experiment.py experiment --load MyCheckpoint\"", "response": "List available checkpoints for the specified experiment."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef run(self):\n    self.__logger.debug(\"run(): Starting task <%s>\", self.__task['taskLabel'])\n\n    # Set up the task\n\n    # Create our main loop-control iterator\n    if self.__cmdOptions.privateOptions['testMode']:\n      numIters = 10\n    else:\n      numIters = self.__task['iterationCount']\n\n    if numIters >= 0:\n      iterTracker = iter(xrange(numIters))\n    else:\n      iterTracker = iter(itertools.count())\n\n    # Initialize periodic activities\n    periodic = PeriodicActivityMgr(\n      requestedActivities=self._createPeriodicActivities())\n\n    # Reset sequence states in the model, so it starts looking for a new\n    # sequence\n    # TODO: should this be done in OPFTaskDriver.setup(), instead?  Is it always\n    #       desired in Nupic?\n    self.__model.resetSequenceStates()\n\n    # Have Task Driver perform its initial setup activities, including setup\n    # callbacks\n    self.__taskDriver.setup()\n\n    # Run it!\n    while True:\n      # Check controlling iterator first\n      try:\n        next(iterTracker)\n      except StopIteration:\n        break\n\n      # Read next input record\n      try:\n        inputRecord = self.__datasetReader.next()\n      except StopIteration:\n        break\n\n      # Process input record\n      result = self.__taskDriver.handleInputRecord(inputRecord=inputRecord)\n\n      if InferenceElement.encodings in result.inferences:\n        result.inferences.pop(InferenceElement.encodings)\n      self.__predictionLogger.writeRecord(result)\n\n      # Run periodic activities\n      periodic.tick()\n\n    # Dump the experiment metrics at the end of the task\n    self._getAndEmitExperimentMetrics(final=True)\n\n    # Have Task Driver perform its final activities\n    self.__taskDriver.finalize()\n\n    # Reset sequence states in the model, so it starts looking for a new\n    # sequence\n    # TODO: should this be done in OPFTaskDriver.setup(), instead?  Is it always\n    #       desired in Nupic?\n    self.__model.resetSequenceStates()", "response": "Runs a single experiment task"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating and returns a list of PeriodicActivityRequest elements for this TaskRunner instance", "response": "def _createPeriodicActivities(self):\n    \"\"\"Creates and returns a list of activites for this TaskRunner instance\n\n    Returns: a list of PeriodicActivityRequest elements\n    \"\"\"\n    # Initialize periodic activities\n    periodicActivities = []\n\n    # Metrics reporting\n    class MetricsReportCb(object):\n      def __init__(self, taskRunner):\n        self.__taskRunner = taskRunner\n        return\n\n      def __call__(self):\n        self.__taskRunner._getAndEmitExperimentMetrics()\n\n    reportMetrics = PeriodicActivityRequest(\n      repeating=True,\n      period=1000,\n      cb=MetricsReportCb(self))\n\n    periodicActivities.append(reportMetrics)\n\n    # Iteration progress\n    class IterationProgressCb(object):\n      PROGRESS_UPDATE_PERIOD_TICKS = 1000\n\n      def __init__(self, taskLabel, requestedIterationCount, logger):\n        self.__taskLabel = taskLabel\n        self.__requestedIterationCount = requestedIterationCount\n        self.__logger = logger\n\n        self.__numIterationsSoFar = 0\n\n      def __call__(self):\n        self.__numIterationsSoFar += self.PROGRESS_UPDATE_PERIOD_TICKS\n        self.__logger.debug(\"%s: ITERATION PROGRESS: %s of %s\" % (\n                              self.__taskLabel,\n                              self.__numIterationsSoFar,\n                              self.__requestedIterationCount))\n\n    iterationProgressCb = IterationProgressCb(\n      taskLabel=self.__task['taskLabel'],\n      requestedIterationCount=self.__task['iterationCount'],\n      logger=self.__logger)\n    iterationProgressReporter = PeriodicActivityRequest(\n      repeating=True,\n      period=IterationProgressCb.PROGRESS_UPDATE_PERIOD_TICKS,\n      cb=iterationProgressCb)\n\n    periodicActivities.append(iterationProgressReporter)\n\n    return periodicActivities"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _generateFile(filename, data):\n  \n  # Create the file\n  print \"Creating %s...\" % (filename)\n  numRecords, numFields = data.shape\n  \n  fields = [('field%d'%(i+1), 'float', '') for i in range(numFields)]\n  outFile = File(filename, fields)\n  \n  for i in xrange(numRecords):\n    outFile.write(data[i].tolist())\n    \n  outFile.close()", "response": "Generates the file for the sequence number."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef corruptVector(v1, noiseLevel, numActiveCols):\n  size = len(v1)\n  v2 = np.zeros(size, dtype=\"uint32\")\n  bitsToSwap = int(noiseLevel * numActiveCols)\n  # Copy the contents of v1 into v2\n  for i in range(size):\n    v2[i] = v1[i]\n  for _ in range(bitsToSwap):\n    i = random.randrange(size)\n    if v2[i] == 1:\n      v2[i] = 0\n    else:\n      v2[i] = 1\n  return v2", "response": "Corrupts a copy of a binary vector by inverting noiseLevel percent of its bits."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nshow predictions of the TM when presented with the characters A B C D X and Y without any contextual information that is embedded within a sequence.", "response": "def showPredictions():\n  \"\"\"\n  Shows predictions of the TM when presented with the characters A, B, C, D, X, and\n  Y without any contextual information, that is, not embedded within a sequence.\n  \"\"\"   \n  for k in range(6):\n    tm.reset()\n    print \"--- \" + \"ABCDXY\"[k] + \" ---\"\n    tm.compute(set(seqT[k][:].nonzero()[0].tolist()), learn=False)\n    activeColumnsIndices = [tm.columnForCell(i) for i in tm.getActiveCells()]\n    predictedColumnIndices = [tm.columnForCell(i) for i in tm.getPredictiveCells()]  \n    currentColumns = [1 if i in activeColumnsIndices else 0 for i in range(tm.numberOfColumns())]\n    predictedColumns = [1 if i in predictedColumnIndices else 0 for i in range(tm.numberOfColumns())]\n    print(\"Active cols: \" + str(np.nonzero(currentColumns)[0]))\n    print(\"Predicted cols: \" + str(np.nonzero(predictedColumns)[0]))\n    print \"\""}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ntrain the TM with given sequence for a given number of time steps and level of input", "response": "def trainTM(sequence, timeSteps, noiseLevel):\n  \"\"\"\n  Trains the TM with given sequence for a given number of time steps and level of input\n  corruption\n  \n  @param sequence   (array) array whose rows are the input characters\n  @param timeSteps  (int)   number of time steps in which the TM will be presented with sequence\n  @param noiseLevel (float) amount of noise to be applied on the characters in the sequence \n  \"\"\"\n  currentColumns = np.zeros(tm.numberOfColumns(), dtype=\"uint32\")\n  predictedColumns = np.zeros(tm.numberOfColumns(), dtype=\"uint32\")\n  ts = 0  \n  for t in range(timeSteps):\n    tm.reset()\n    for k in range(4):\n      v = corruptVector(sequence[k][:], noiseLevel, sparseCols)\n      tm.compute(set(v[:].nonzero()[0].tolist()), learn=True)\n      activeColumnsIndices = [tm.columnForCell(i) for i in tm.getActiveCells()]\n      predictedColumnIndices = [tm.columnForCell(i) for i in tm.getPredictiveCells()]\n      currentColumns = [1 if i in activeColumnsIndices else 0 for i in range(tm.numberOfColumns())]\n      acc = accuracy(currentColumns, predictedColumns)\n      x.append(ts)\n      y.append(acc)\n      ts += 1\n      predictedColumns = [1 if i in predictedColumnIndices else 0 for i in range(tm.numberOfColumns())]"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef encodeIntoArray(self, inputVal, outputVal):\n    if len(inputVal) != len(outputVal):\n      raise ValueError(\"Different input (%i) and output (%i) sizes.\" % (\n          len(inputVal), len(outputVal)))\n\n    if self.w is not None and sum(inputVal) != self.w:\n      raise ValueError(\"Input has %i bits but w was set to %i.\" % (\n          sum(inputVal), self.w))\n\n    outputVal[:] = inputVal[:]\n\n    if self.verbosity >= 2:\n      print \"input:\", inputVal, \"output:\", outputVal\n      print \"decoded:\", self.decodedToStr(self.decode(outputVal))", "response": "Encodes inputVal into outputVal."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef decode(self, encoded, parentFieldName=\"\"):\n\n    if parentFieldName != \"\":\n      fieldName = \"%s.%s\" % (parentFieldName, self.name)\n    else:\n      fieldName = self.name\n\n    return ({fieldName: ([[0, 0]], \"input\")}, [fieldName])", "response": "Decode the encoded field into a dictionary."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsee the function description in base. py", "response": "def getBucketInfo(self, buckets):\n    \"\"\"See the function description in base.py\"\"\"\n    return [EncoderResult(value=0, scalar=0, encoding=numpy.zeros(self.n))]"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef topDownCompute(self, encoded):\n    return EncoderResult(value=0, scalar=0,\n                         encoding=numpy.zeros(self.n))", "response": "Compute the top down encoding of a single element."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndoing a bitwise compare of the two bitmaps and returns a fractonal value between 0 and 1 of how similar they are.", "response": "def closenessScores(self, expValues, actValues, **kwargs):\n    \"\"\"\n    Does a bitwise compare of the two bitmaps and returns a fractonal\n    value between 0 and 1 of how similar they are.\n\n    - ``1`` => identical\n    - ``0`` => no overlaping bits\n\n    ``kwargs`` will have the keyword \"fractional\", which is assumed by this\n    encoder.\n    \"\"\"\n    ratio = 1.0\n    esum = int(expValues.sum())\n    asum = int(actValues.sum())\n    if asum > esum:\n      diff = asum - esum\n      if diff < esum:\n        ratio = 1 - diff/float(esum)\n      else:\n        ratio = 1/float(diff)\n\n    olap = expValues & actValues\n    osum = int(olap.sum())\n    if esum == 0:\n      r = 0.0\n    else:\n      r = osum/float(esum)\n    r = r * ratio\n\n    return numpy.array([r])"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getCallerInfo(depth=2):\n  f = sys._getframe(depth)\n  method_name = f.f_code.co_name\n  filename = f.f_code.co_filename\n\n  arg_class = None\n  args = inspect.getargvalues(f)\n  if len(args[0]) > 0:\n    arg_name = args[0][0] # potentially the 'self' arg if its a method\n    arg_class = args[3][arg_name].__class__.__name__\n  return (method_name, filename, arg_class)", "response": "Utility function to get information about the caller s caller\n        "}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the argument names default values and argument descriptions for a function.", "response": "def getArgumentDescriptions(f):\n  \"\"\"\n  Get the arguments, default values, and argument descriptions for a function.\n\n  Parses the argument descriptions out of the function docstring, using a\n  format something lke this:\n\n  ::\n\n    [junk]\n    argument_name:     description...\n      description...\n      description...\n    [junk]\n    [more arguments]\n\n  It will find an argument as long as the exact argument name starts the line.\n  It will then strip a trailing colon, if present, then strip the rest of the\n  line and use it to start the description. It will then strip and append any\n  subsequent lines with a greater indent level than the original argument name.\n\n  :param f: (function) to inspect\n  :returns: (list of tuples) (``argName``, ``argDescription``, ``defaultValue``)\n    If an argument has no default value, the tuple is only two elements long (as\n    ``None`` cannot be used, since it could be a default value itself).\n  \"\"\"\n\n  # Get the argument names and default values\n  argspec = inspect.getargspec(f)\n\n  # Scan through the docstring to extract documentation for each argument as\n  # follows:\n  #   Check the first word of the line, stripping a colon if one is present.\n  #   If it matches an argument name:\n  #    Take the rest of the line, stripping leading whitespeace\n  #    Take each subsequent line if its indentation level is greater than the\n  #      initial indentation level\n  #    Once the indentation level is back to the original level, look for\n  #      another argument\n  docstring = f.__doc__\n  descriptions = {}\n  if docstring:\n    lines = docstring.split('\\n')\n    i = 0\n    while i < len(lines):\n      stripped = lines[i].lstrip()\n      if not stripped:\n        i += 1\n        continue\n      # Indentation level is index of the first character\n      indentLevel = lines[i].index(stripped[0])\n      # Get the first word and remove the colon, if present\n      firstWord = stripped.split()[0]\n      if firstWord.endswith(':'):\n        firstWord = firstWord[:-1]\n      if firstWord in argspec.args:\n        # Found an argument\n        argName = firstWord\n        restOfLine = stripped[len(firstWord)+1:].strip()\n        argLines = [restOfLine]\n        # Take the next lines as long as they are indented more\n        i += 1\n        while i < len(lines):\n          stripped = lines[i].lstrip()\n          if not stripped:\n            # Empty line - stop\n            break\n          if lines[i].index(stripped[0]) <= indentLevel:\n            # No longer indented far enough - stop\n            break\n          # This line counts too\n          argLines.append(lines[i].strip())\n          i += 1\n        # Store this description\n        descriptions[argName] = ' '.join(argLines)\n      else:\n        # Not an argument\n        i += 1\n\n  # Build the list of (argName, description, defaultValue)\n  args = []\n  if argspec.defaults:\n    defaultCount = len(argspec.defaults)\n  else:\n    defaultCount = 0\n  nonDefaultArgCount = len(argspec.args) - defaultCount\n  for i, argName in enumerate(argspec.args):\n    if i >= nonDefaultArgCount:\n      defaultValue = argspec.defaults[i - nonDefaultArgCount]\n      args.append((argName, descriptions.get(argName, \"\"), defaultValue))\n    else:\n      args.append((argName, descriptions.get(argName, \"\")))\n\n  return args"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef initLogging(verbose=False, console='stdout', consoleLevel='DEBUG'):\n\n  # NOTE: If you call this twice from the same process there seems to be a\n  # bug - logged messages don't show up for loggers that you do another\n  # logging.getLogger() on.\n  global gLoggingInitialized\n  if gLoggingInitialized:\n    if verbose:\n      print >> sys.stderr, \"Logging already initialized, doing nothing.\"\n    return\n\n  consoleStreamMappings = {\n    'stdout'  : 'stdoutConsoleHandler',\n    'stderr'  : 'stderrConsoleHandler',\n  }\n\n  consoleLogLevels = ['DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL',\n                      'FATAL']\n\n  assert console is None or console in consoleStreamMappings.keys(), (\n    'Unexpected console arg value: %r') % (console,)\n\n  assert consoleLevel in consoleLogLevels, (\n    'Unexpected consoleLevel arg value: %r') % (consoleLevel)\n\n  # -----------------------------------------------------------------------\n  # Setup logging. Look for the nupic-logging.conf file, first in the\n  #   NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic\n  #   module\n  configFilename = 'nupic-logging.conf'\n  configFilePath = resource_filename(\"nupic.support\", configFilename)\n\n  configLogDir = os.environ.get('NTA_LOG_DIR', None)\n\n  # Load in the logging configuration file\n  if verbose:\n    print >> sys.stderr, (\n      \"Using logging configuration file: %s\") % (configFilePath)\n\n  # This dict will hold our replacement strings for logging configuration\n  replacements = dict()\n\n  def makeKey(name):\n    \"\"\" Makes replacement key \"\"\"\n    return \"$$%s$$\" % (name)\n\n  platform = sys.platform.lower()\n  if platform.startswith('java'):\n    # Jython\n    import java.lang\n    platform = java.lang.System.getProperty(\"os.name\").lower()\n    if platform.startswith('mac os x'):\n      platform = 'darwin'\n\n  if platform.startswith('darwin'):\n    replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '\"/var/run/syslog\"'\n  elif platform.startswith('linux'):\n    replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '\"/dev/log\"'\n  elif platform.startswith('win'):\n    replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '\"log\"'\n  else:\n    raise RuntimeError(\"This platform is neither darwin, win32, nor linux: %s\" % (\n      sys.platform,))\n\n  # Nupic logs go to file\n  replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'fileHandler'\n  if platform.startswith('win'):\n    replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = '\"NUL\"'\n  else:\n    replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = '\"/dev/null\"'\n\n  # Set up log file path for the default file handler and configure handlers\n  handlers = list()\n\n  if configLogDir is not None:\n    logFilePath = _genLoggingFilePath()\n    makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath))\n    replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = repr(logFilePath)\n\n    handlers.append(replacements[makeKey('PERSISTENT_LOG_HANDLER')])\n\n  if console is not None:\n    handlers.append(consoleStreamMappings[console])\n\n  replacements[makeKey('ROOT_LOGGER_HANDLERS')] = \", \".join(handlers)\n\n  # Set up log level for console handlers\n  replacements[makeKey('CONSOLE_LOG_LEVEL')] = consoleLevel\n\n  customConfig = StringIO()\n\n  # Using pkg_resources to get the logging file, which should be packaged and\n  # associated with this source file name.\n  loggingFileContents = resource_string(__name__, configFilename)\n\n  for lineNum, line in enumerate(loggingFileContents.splitlines()):\n    if \"$$\" in line:\n      for (key, value) in replacements.items():\n        line = line.replace(key, value)\n\n    # If there is still a replacement string in the line, we're missing it\n    #  from our replacements dict\n    if \"$$\" in line and \"$$<key>$$\" not in line:\n      raise RuntimeError((\"The text %r, found at line #%d of file %r, \"\n                          \"contains a string not found in our replacement \"\n                          \"dict.\") % (line, lineNum, configFilePath))\n\n    customConfig.write(\"%s\\n\" % line)\n\n  customConfig.seek(0)\n  if python_version()[:3] >= '2.6':\n    logging.config.fileConfig(customConfig, disable_existing_loggers=False)\n  else:\n    logging.config.fileConfig(customConfig)\n\n  gLoggingInitialized = True", "response": "Initializes logging for the NuPic."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _genLoggingFilePath():\n  appName = os.path.splitext(os.path.basename(sys.argv[0]))[0] or 'UnknownApp'\n  appLogDir = os.path.abspath(os.path.join(\n    os.environ['NTA_LOG_DIR'],\n    'numenta-logs-%s' % (os.environ['USER'],),\n    appName))\n  appLogFileName = '%s-%s-%s.log' % (\n    appName, long(time.mktime(time.gmtime())), os.getpid())\n  return os.path.join(appLogDir, appLogFileName)", "response": "Generate a filepath for the calling app"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the number of months and seconds from an aggregation dict that represents a date and time. Interval is a dict that contain one or more of the following keys: 'years', 'months', 'weeks', 'days', 'hours', 'minutes', seconds', 'milliseconds', 'microseconds'. For example: :: aggregationMicroseconds({'years': 1, 'hours': 4, 'microseconds':42}) == {'months':12, 'seconds':14400.000042} :param interval: (dict) The aggregation interval representing a date and time :returns: (dict) number of months and seconds in the interval: ``{months': XX, 'seconds': XX}``. The seconds is a floating point that can represent resolutions down to a microsecond.", "response": "def aggregationToMonthsSeconds(interval):\n  \"\"\"\n  Return the number of months and seconds from an aggregation dict that\n  represents a date and time.\n\n  Interval is a dict that contain one or more of the following keys: 'years',\n  'months', 'weeks', 'days', 'hours', 'minutes', seconds', 'milliseconds',\n  'microseconds'.\n\n  For example:\n\n  ::\n\n    aggregationMicroseconds({'years': 1, 'hours': 4, 'microseconds':42}) ==\n        {'months':12, 'seconds':14400.000042}\n\n  :param interval: (dict) The aggregation interval representing a date and time\n  :returns: (dict) number of months and seconds in the interval:\n            ``{months': XX, 'seconds': XX}``. The seconds is\n            a floating point that can represent resolutions down to a\n            microsecond.\n\n  \"\"\"\n\n  seconds = interval.get('microseconds', 0) * 0.000001\n  seconds += interval.get('milliseconds', 0) * 0.001\n  seconds += interval.get('seconds', 0)\n  seconds += interval.get('minutes', 0) * 60\n  seconds += interval.get('hours', 0) * 60 * 60\n  seconds += interval.get('days', 0) * 24 * 60 * 60\n  seconds += interval.get('weeks', 0) * 7 * 24 * 60 * 60\n\n  months = interval.get('months', 0)\n  months += 12 * interval.get('years', 0)\n\n  return {'months': months, 'seconds': seconds}"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef aggregationDivide(dividend, divisor):\n\n  # Convert each into microseconds\n  dividendMonthSec = aggregationToMonthsSeconds(dividend)\n  divisorMonthSec = aggregationToMonthsSeconds(divisor)\n\n  # It is a usage error to mix both months and seconds in the same operation\n  if (dividendMonthSec['months'] != 0 and divisorMonthSec['seconds'] != 0) \\\n    or (dividendMonthSec['seconds'] != 0 and divisorMonthSec['months'] != 0):\n    raise RuntimeError(\"Aggregation dicts with months/years can only be \"\n      \"inter-operated with other aggregation dicts that contain \"\n      \"months/years\")\n\n\n  if dividendMonthSec['months'] > 0:\n    return float(dividendMonthSec['months']) / divisor['months']\n\n  else:\n    return float(dividendMonthSec['seconds']) / divisorMonthSec['seconds']", "response": "This function takes two dicts representing a date and time and returns the result from dividing the two dicts representing a date and time."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef validateOpfJsonValue(value, opfJsonSchemaFilename):\n\n  # Create a path by joining the filename with our local json schema root\n  jsonSchemaPath = os.path.join(os.path.dirname(__file__),\n                                \"jsonschema\",\n                                opfJsonSchemaFilename)\n\n  # Validate\n  jsonhelpers.validate(value, schemaPath=jsonSchemaPath)\n\n  return", "response": "Validate a python object against an OPF json schema file."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef matchPatterns(patterns, keys):\n  results = []\n  if patterns:\n    for pattern in patterns:\n      prog = re.compile(pattern)\n      for key in keys:\n        if prog.match(key):\n          results.append(key)\n  else:\n    return None\n\n  return results", "response": "Returns a subset of the keys that match any of the given patterns\n\n."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _getScaledValue(self, inpt):\n    if inpt == SENTINEL_VALUE_FOR_MISSING_DATA:\n      return None\n    else:\n      val = inpt\n      if val < self.minval:\n        val = self.minval\n      elif val > self.maxval:\n        val = self.maxval\n\n      scaledVal = math.log10(val)\n      return scaledVal", "response": "Convert the input which is in normal space into log10"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getBucketIndices(self, inpt):\n\n    # Get the scaled value\n    scaledVal = self._getScaledValue(inpt)\n\n    if scaledVal is None:\n      return [None]\n    else:\n      return self.encoder.getBucketIndices(scaledVal)", "response": "Returns the indices of the bucket in the input value."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nencoding the input into an array.", "response": "def encodeIntoArray(self, inpt, output):\n    \"\"\"\n    See the function description in base.py\n    \"\"\"\n\n    # Get the scaled value\n    scaledVal = self._getScaledValue(inpt)\n\n    if scaledVal is None:\n      output[0:] = 0\n    else:\n      self.encoder.encodeIntoArray(scaledVal, output)\n\n      if self.verbosity >= 2:\n        print \"input:\", inpt, \"scaledVal:\", scaledVal, \"output:\", output\n        print \"decoded:\", self.decodedToStr(self.decode(output))"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef decode(self, encoded, parentFieldName=''):\n\n    # Get the scalar values from the underlying scalar encoder\n    (fieldsDict, fieldNames) = self.encoder.decode(encoded)\n    if len(fieldsDict) == 0:\n      return (fieldsDict, fieldNames)\n\n    # Expect only 1 field\n    assert(len(fieldsDict) == 1)\n\n    # Convert each range into normal space\n    (inRanges, inDesc) = fieldsDict.values()[0]\n    outRanges = []\n    for (minV, maxV) in inRanges:\n      outRanges.append((math.pow(10, minV),\n                        math.pow(10, maxV)))\n\n    # Generate a text description of the ranges\n    desc = \"\"\n    numRanges = len(outRanges)\n    for i in xrange(numRanges):\n      if outRanges[i][0] != outRanges[i][1]:\n        desc += \"%.2f-%.2f\" % (outRanges[i][0], outRanges[i][1])\n      else:\n        desc += \"%.2f\" % (outRanges[i][0])\n      if i < numRanges-1:\n        desc += \", \"\n\n    # Return result\n    if parentFieldName != '':\n      fieldName = \"%s.%s\" % (parentFieldName, self.name)\n    else:\n      fieldName = self.name\n    return ({fieldName: (outRanges, desc)}, [fieldName])", "response": "Decode the encoded string into a single object."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a list of all the bucket values in the current language.", "response": "def getBucketValues(self):\n    \"\"\"\n    See the function description in base.py\n    \"\"\"\n\n    # Need to re-create?\n    if self._bucketValues is None:\n      scaledValues = self.encoder.getBucketValues()\n      self._bucketValues = []\n      for scaledValue in scaledValues:\n        value = math.pow(10, scaledValue)\n        self._bucketValues.append(value)\n\n    return self._bucketValues"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning encoderResult for the given buckets.", "response": "def getBucketInfo(self, buckets):\n    \"\"\"\n    See the function description in base.py\n    \"\"\"\n\n    scaledResult = self.encoder.getBucketInfo(buckets)[0]\n    scaledValue = scaledResult.value\n    value = math.pow(10, scaledValue)\n\n    return [EncoderResult(value=value, scalar=value,\n                         encoding = scaledResult.encoding)]"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomputing the top down encoding of the encoded sequence.", "response": "def topDownCompute(self, encoded):\n    \"\"\"\n    See the function description in base.py\n    \"\"\"\n\n    scaledResult = self.encoder.topDownCompute(encoded)[0]\n    scaledValue = scaledResult.value\n    value = math.pow(10, scaledValue)\n\n    return EncoderResult(value=value, scalar=value,\n                         encoding = scaledResult.encoding)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef closenessScores(self, expValues, actValues, fractional=True):\n\n    # Compute the percent error in log space\n    if expValues[0] > 0:\n      expValue = math.log10(expValues[0])\n    else:\n      expValue = self.minScaledValue\n\n    if actValues  [0] > 0:\n      actValue = math.log10(actValues[0])\n    else:\n      actValue = self.minScaledValue\n\n    if fractional:\n      err = abs(expValue - actValue)\n      pctErr = err / (self.maxScaledValue - self.minScaledValue)\n      pctErr = min(1.0, pctErr)\n      closeness = 1.0 - pctErr\n    else:\n      err = abs(expValue - actValue)\n      closeness = err\n\n    #print \"log::\", \"expValue:\", expValues[0], \"actValue:\", actValues[0], \\\n    #      \"closeness\", closeness\n    #import pdb; pdb.set_trace()\n    return numpy.array([closeness])", "response": "Compute the closeness scores of the given values."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef export(self):\n    graph = nx.MultiDiGraph()\n\n    # Add regions to graph as nodes, annotated by name\n    regions = self.network.getRegions()\n\n    for idx in xrange(regions.getCount()):\n      regionPair = regions.getByIndex(idx)\n      regionName = regionPair[0]\n      graph.add_node(regionName, label=regionName)\n\n    # Add links between regions to graph as edges, annotate by input-output\n    # name pairs\n    for linkName, link in self.network.getLinks():\n      graph.add_edge(link.getSrcRegionName(),\n                     link.getDestRegionName(),\n                     src=link.getSrcOutputName(),\n                     dest=link.getDestInputName())\n\n    return graph", "response": "Exports a network as a networkx MultiDiGraph intermediate representation of the network."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a string representing a numpy array of 0 s and 1 s", "response": "def bitsToString(arr):\n  \"\"\"Returns a string representing a numpy array of 0's and 1's\"\"\"\n  s = array('c','.'*len(arr))\n  for i in xrange(len(arr)):\n    if arr[i] == 1:\n      s[i]='*'\n  return s"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompute the percentage of overlap between vectors x1 and x2.", "response": "def percentOverlap(x1, x2, size):\n  \"\"\"\n  Computes the percentage of overlap between vectors x1 and x2.\n\n  @param x1   (array) binary vector\n  @param x2   (array) binary vector\n  @param size (int)   length of binary vectors\n\n  @return percentOverlap (float) percentage overlap between x1 and x2\n  \"\"\"\n  nonZeroX1 = np.count_nonzero(x1)\n  nonZeroX2 = np.count_nonzero(x2)\n  minX1X2 = min(nonZeroX1, nonZeroX2)\n  percentOverlap = 0\n  if minX1X2 > 0:\n    percentOverlap = float(np.dot(x1, x2))/float(minX1X2)\n  return percentOverlap"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nresetting the contents of vector x1 into vector x2.", "response": "def resetVector(x1, x2):\n  \"\"\"\n  Copies the contents of vector x1 into vector x2.\n\n  @param x1 (array) binary vector to be copied\n  @param x2 (array) binary vector where x1 is copied\n  \"\"\"\n  size = len(x1)\n  for i in range(size):\n    x2[i] = x1[i]"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\npoll CPU usage make predictions and plot the results. Runs forever.", "response": "def runCPU():\n  \"\"\"Poll CPU usage, make predictions, and plot the results. Runs forever.\"\"\"\n  # Create the model for predicting CPU usage.\n  model = ModelFactory.create(model_params.MODEL_PARAMS)\n  model.enableInference({'predictedField': 'cpu'})\n  # The shifter will align prediction and actual values.\n  shifter = InferenceShifter()\n  # Keep the last WINDOW predicted and actual values for plotting.\n  actHistory = deque([0.0] * WINDOW, maxlen=60)\n  predHistory = deque([0.0] * WINDOW, maxlen=60)\n\n  # Initialize the plot lines that we will update with each new record.\n  actline, = plt.plot(range(WINDOW), actHistory)\n  predline, = plt.plot(range(WINDOW), predHistory)\n  # Set the y-axis range.\n  actline.axes.set_ylim(0, 100)\n  predline.axes.set_ylim(0, 100)\n\n  while True:\n    s = time.time()\n\n    # Get the CPU usage.\n    cpu = psutil.cpu_percent()\n\n    # Run the input through the model and shift the resulting prediction.\n    modelInput = {'cpu': cpu}\n    result = shifter.shift(model.run(modelInput))\n\n    # Update the trailing predicted and actual value deques.\n    inference = result.inferences['multiStepBestPredictions'][5]\n    if inference is not None:\n      actHistory.append(result.rawInput['cpu'])\n      predHistory.append(inference)\n\n    # Redraw the chart with the new data.\n    actline.set_ydata(actHistory)  # update the data\n    predline.set_ydata(predHistory)  # update the data\n    plt.draw()\n    plt.legend( ('actual','predicted') )\n\n    # Make sure we wait a total of 2 seconds per iteration.\n    try:\n      plt.pause(SECONDS_PER_STEP)\n    except:\n      pass"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a dictionary of the args dictionary from the calling method", "response": "def _extractCallingMethodArgs():\n  \"\"\"\n  Returns args dictionary from the calling method\n  \"\"\"\n  import inspect\n  import copy\n\n  callingFrame = inspect.stack()[1][0]\n\n  argNames, _, _, frameLocalVarDict = inspect.getargvalues(callingFrame)\n\n  argNames.remove(\"self\")\n\n  args = copy.copy(frameLocalVarDict)\n\n\n  for varName in frameLocalVarDict:\n    if varName not in argNames:\n      args.pop(varName)\n\n  return args"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\npopulates serialization proto instance.", "response": "def write(self, proto):\n    \"\"\"Populate serialization proto instance.\n\n    :param proto: (BacktrackingTMCppProto) the proto instance to populate\n    \"\"\"\n    # Write base class to proto.baseTM (BacktrackingTMProto)\n    super(BacktrackingTMCPP, self).write(proto.baseTM)\n    self.cells4.write(proto.cells4)\n    proto.makeCells4Ephemeral = self.makeCells4Ephemeral\n    proto.seed = self.seed\n    proto.checkSynapseConsistency = self.checkSynapseConsistency\n    proto.initArgs = json.dumps(self._initArgsDict)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef read(cls, proto):\n    # Use base class to create initial class from proto.baseTM\n    # (BacktrackingTMProto)\n    obj = BacktrackingTM.read(proto.baseTM)\n    obj.__class__ = cls\n\n    # Additional CPP-specific deserialization\n    newCells4 = Cells4.read(proto.cells4)\n    print newCells4\n    obj.cells4 = newCells4\n    obj.makeCells4Ephemeral = proto.makeCells4Ephemeral\n    obj.seed = proto.seed\n    obj.checkSynapseConsistency = proto.checkSynapseConsistency\n    obj._initArgsDict = json.loads(proto.initArgs)\n    # Convert unicode to str\n    obj._initArgsDict[\"outputType\"] = str(obj._initArgsDict[\"outputType\"])\n\n    # Initialize ephemeral attributes\n    obj.allocateStatesInCPP = False\n    obj.retrieveLearningStates = False\n    obj._setStatePointers()\n\n    return obj", "response": "Deserialize from proto instance."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _getEphemeralMembers(self):\n    e = BacktrackingTM._getEphemeralMembers(self)\n    if self.makeCells4Ephemeral:\n      e.extend(['cells4'])\n    return e", "response": "Returns a list of our member variables that we don t need to be saved"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _initEphemerals(self):\n    BacktrackingTM._initEphemerals(self)\n    #---------------------------------------------------------------------------------\n    # cells4 specific initialization\n\n    # If True, let C++ allocate memory for activeState, predictedState, and\n    # learnState. In this case we can retrieve copies of these states but can't\n    # set them directly from Python. If False, Python can allocate them as\n    # numpy arrays and we can pass pointers to the C++ using setStatePointers\n    self.allocateStatesInCPP = False\n\n    # Set this to true for debugging or accessing learning states\n    self.retrieveLearningStates = False\n\n    if self.makeCells4Ephemeral:\n      self._initCells4()", "response": "Initialize all ephemeral members after being restored to a pickled state."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef compute(self, bottomUpInput, enableLearn, enableInference=None):\n    # The C++ TM takes 32 bit floats as input. uint32 works as well since the\n    # code only checks whether elements are non-zero\n    assert (bottomUpInput.dtype == numpy.dtype('float32')) or \\\n           (bottomUpInput.dtype == numpy.dtype('uint32')) or \\\n           (bottomUpInput.dtype == numpy.dtype('int32'))\n\n    self.iterationIdx = self.iterationIdx + 1\n\n    # As a speed optimization for now (until we need online learning), skip\n    #  computing the inference output while learning\n    if enableInference is None:\n      if enableLearn:\n        enableInference = False\n      else:\n        enableInference = True\n\n    # ====================================================================\n    # Run compute and retrieve selected state and member variables\n    self._setStatePointers()\n    y = self.cells4.compute(bottomUpInput, enableInference, enableLearn)\n    self.currentOutput = y.reshape((self.numberOfCols, self.cellsPerColumn))\n    self.avgLearnedSeqLength = self.cells4.getAvgLearnedSeqLength()\n    self._copyAllocatedStates()\n\n\n    # ========================================================================\n    # Update the prediction score stats\n    # Learning always includes inference\n    if self.collectStats:\n      activeColumns = bottomUpInput.nonzero()[0]\n      if enableInference:\n        predictedState = self.infPredictedState['t-1']\n      else:\n        predictedState = self.lrnPredictedState['t-1']\n      self._updateStatsInferEnd(self._internalStats,\n                                activeColumns,\n                                predictedState,\n                                self.colConfidence['t-1'])\n\n\n\n    # Finally return the TM output\n    output = self._computeOutput()\n\n    # Print diagnostic information based on the current verbosity level\n    self.printComputeEnd(output, learn=enableLearn)\n\n    self.resetCalled = False\n    return output", "response": "Compute the current state of the C ++ TM."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _copyAllocatedStates(self):\n\n    # Get learn states if we need to print them out\n    if self.verbosity > 1 or self.retrieveLearningStates:\n      (activeT, activeT1, predT, predT1) = self.cells4.getLearnStates()\n      self.lrnActiveState['t-1'] = activeT1.reshape((self.numberOfCols, self.cellsPerColumn))\n      self.lrnActiveState['t'] = activeT.reshape((self.numberOfCols, self.cellsPerColumn))\n      self.lrnPredictedState['t-1'] = predT1.reshape((self.numberOfCols, self.cellsPerColumn))\n      self.lrnPredictedState['t'] = predT.reshape((self.numberOfCols, self.cellsPerColumn))\n\n    if self.allocateStatesInCPP:\n      assert False\n      (activeT, activeT1, predT, predT1, colConfidenceT, colConfidenceT1, confidenceT,\n       confidenceT1) = self.cells4.getStates()\n      self.cellConfidence['t'] = confidenceT.reshape((self.numberOfCols, self.cellsPerColumn))\n      self.cellConfidence['t-1'] = confidenceT1.reshape((self.numberOfCols, self.cellsPerColumn))\n      self.colConfidence['t'] = colConfidenceT.reshape(self.numberOfCols)\n      self.colConfidence['t-1'] = colConfidenceT1.reshape(self.numberOfCols)\n      self.infActiveState['t-1'] = activeT1.reshape((self.numberOfCols, self.cellsPerColumn))\n      self.infActiveState['t'] = activeT.reshape((self.numberOfCols, self.cellsPerColumn))\n      self.infPredictedState['t-1'] = predT1.reshape((self.numberOfCols, self.cellsPerColumn))\n      self.infPredictedState['t'] = predT.reshape((self.numberOfCols, self.cellsPerColumn))", "response": "Copy the allocated states into our numpy arrays."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsets the state pointers for the cells4 object.", "response": "def _setStatePointers(self):\n    \"\"\"If we are having CPP use numpy-allocated buffers, set these buffer\n    pointers. This is a relatively fast operation and, for safety, should be\n    done before every call to the cells4 compute methods.  This protects us\n    in situations where code can cause Python or numpy to create copies.\"\"\"\n    if not self.allocateStatesInCPP:\n      self.cells4.setStatePointers(\n          self.infActiveState[\"t\"], self.infActiveState[\"t-1\"],\n          self.infPredictedState[\"t\"], self.infPredictedState[\"t-1\"],\n          self.colConfidence[\"t\"], self.colConfidence[\"t-1\"],\n          self.cellConfidence[\"t\"], self.cellConfidence[\"t-1\"])"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a new sequence of cells with only the given synapses.", "response": "def trimSegments(self, minPermanence=None, minNumSyns=None):\n    \"\"\"\n    Overrides :meth:`nupic.algorithms.backtracking_tm.BacktrackingTM.trimSegments`.\n    \"\"\"\n    # Fill in defaults\n    if minPermanence is None:\n      minPermanence = 0.0\n    if minNumSyns is None:\n      minNumSyns = 0\n\n    # Print all cells if verbosity says to\n    if self.verbosity >= 5:\n      print \"Cells, all segments:\"\n      self.printCells(predictedOnly=False)\n\n    return self.cells4.trimSegments(minPermanence=minPermanence, minNumSyns=minNumSyns)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nprints the segment updates.", "response": "def printSegmentUpdates(self):\n    \"\"\"\n    Overrides :meth:`nupic.algorithms.backtracking_tm.BacktrackingTM.printSegmentUpdates`.\n    \"\"\"\n    # TODO: need to add C++ accessors to implement this method\n    assert False\n    print \"=== SEGMENT UPDATES ===, Num = \", len(self.segmentUpdates)\n    for key, updateList in self.segmentUpdates.iteritems():\n      c,i = key[0],key[1]\n      print c,i,updateList"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef printCell(self, c, i, onlyActiveSegments=False):\n    nSegs = self.cells4.nSegmentsOnCell(c,i)\n    if nSegs > 0:\n      segList = self.cells4.getNonEmptySegList(c,i)\n      gidx = c * self.cellsPerColumn + i\n      print \"Column\", c, \"Cell\", i, \"(%d)\"%(gidx),\":\", nSegs, \"segment(s)\"\n      for k,segIdx in enumerate(segList):\n        seg = self.cells4.getSegment(c, i, segIdx)\n        isActive = self._slowIsSegmentActive(seg, 't')\n        if onlyActiveSegments and not isActive:\n          continue\n        isActiveStr = \"*\" if isActive else \" \"\n        print \"  %sSeg #%-3d\" % (isActiveStr, segIdx),\n        print seg.size(),\n        print seg.isSequenceSegment(), \"%9.7f\" % (seg.dutyCycle(\n              self.cells4.getNLrnIterations(), False, True)),\n\n        # numPositive/totalActivations\n        print \"(%4d/%-4d)\" % (seg.getPositiveActivations(),\n                           seg.getTotalActivations()),\n        # Age\n        print \"%4d\" % (self.cells4.getNLrnIterations()\n                       - seg.getLastActiveIteration()),\n\n        numSyn = seg.size()\n        for s in xrange(numSyn):\n          sc, si = self.getColCellIdx(seg.getSrcCellIdx(s))\n          print \"[%d,%d]%4.2f\"%(sc, si, seg.getPermanence(s)),\n        print", "response": "Prints the contents of a single cell."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getColCellIdx(self, idx):\n    c = idx//self.cellsPerColumn\n    i = idx - c*self.cellsPerColumn\n    return c,i", "response": "Get column and cell within column from a global index."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a list of segments that are on the cell i.", "response": "def getSegmentOnCell(self, c, i, segIdx):\n    \"\"\"\n    Overrides :meth:`nupic.algorithms.backtracking_tm.BacktrackingTM.getSegmentOnCell`.\n    \"\"\"\n    segList = self.cells4.getNonEmptySegList(c,i)\n    seg = self.cells4.getSegment(c, i, segList[segIdx])\n    numSyn = seg.size()\n    assert numSyn != 0\n\n    # Accumulate segment information\n    result = []\n    result.append([int(segIdx), bool(seg.isSequenceSegment()),\n                   seg.getPositiveActivations(),\n                   seg.getTotalActivations(), seg.getLastActiveIteration(),\n                   seg.getLastPosDutyCycle(),\n                   seg.getLastPosDutyCycleIteration()])\n\n    for s in xrange(numSyn):\n      sc, si = self.getColCellIdx(seg.getSrcCellIdx(s))\n      result.append([int(sc), int(si), seg.getPermanence(s)])\n\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a list of all the information for each segment in the current cell.", "response": "def getSegmentInfo(self, collectActiveData = False):\n    \"\"\"\n    Overrides :meth:`nupic.algorithms.backtracking_tm.BacktrackingTM.getSegmentInfo`.\n    \"\"\"\n    # Requires appropriate accessors in C++ cells4 (currently unimplemented)\n    assert collectActiveData == False\n\n    nSegments, nSynapses = self.getNumSegments(), self.cells4.nSynapses()\n    distSegSizes, distNSegsPerCell = {}, {}\n    nActiveSegs, nActiveSynapses = 0, 0\n    distPermValues = {}   # Num synapses with given permanence values\n\n    numAgeBuckets = 20\n    distAges = []\n    ageBucketSize = int((self.iterationIdx+20) / 20)\n    for i in range(numAgeBuckets):\n      distAges.append(['%d-%d' % (i*ageBucketSize, (i+1)*ageBucketSize-1), 0])\n\n\n    for c in xrange(self.numberOfCols):\n      for i in xrange(self.cellsPerColumn):\n\n        # Update histogram counting cell sizes\n        nSegmentsThisCell = self.getNumSegmentsInCell(c,i)\n        if nSegmentsThisCell > 0:\n          if distNSegsPerCell.has_key(nSegmentsThisCell):\n            distNSegsPerCell[nSegmentsThisCell] += 1\n          else:\n            distNSegsPerCell[nSegmentsThisCell] = 1\n\n          # Update histogram counting segment sizes.\n          segList = self.cells4.getNonEmptySegList(c,i)\n          for segIdx in xrange(nSegmentsThisCell):\n            seg = self.getSegmentOnCell(c, i, segIdx)\n            nSynapsesThisSeg = len(seg) - 1\n            if nSynapsesThisSeg > 0:\n              if distSegSizes.has_key(nSynapsesThisSeg):\n                distSegSizes[nSynapsesThisSeg] += 1\n              else:\n                distSegSizes[nSynapsesThisSeg] = 1\n\n              # Accumulate permanence value histogram\n              for syn in seg[1:]:\n                p = int(syn[2]*10)\n                if distPermValues.has_key(p):\n                  distPermValues[p] += 1\n                else:\n                  distPermValues[p] = 1\n\n            segObj = self.cells4.getSegment(c, i, segList[segIdx])\n            age = self.iterationIdx - segObj.getLastActiveIteration()\n            ageBucket = int(age/ageBucketSize)\n            distAges[ageBucket][1] += 1\n\n\n    return (nSegments, nSynapses, nActiveSegs, nActiveSynapses, \\\n            distSegSizes, distNSegsPerCell, distPermValues, distAges)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef main(argv):\n\n  parser = OptionParser(helpString)\n\n  parser.add_option(\"--jobID\", action=\"store\", type=\"int\", default=None,\n        help=\"jobID of the job within the dbTable [default: %default].\")\n\n  parser.add_option(\"--modelID\", action=\"store\", type=\"str\", default=None,\n        help=(\"Tell worker to re-run this model ID. When specified, jobID \"\n         \"must also be specified [default: %default].\"))\n\n  parser.add_option(\"--workerID\", action=\"store\", type=\"str\", default=None,\n        help=(\"workerID of the scheduler's SlotAgent (GenericWorker) that \"\n          \"hosts this SpecializedWorker [default: %default].\"))\n\n  parser.add_option(\"--params\", action=\"store\", default=None,\n        help=\"Create and execute a new hypersearch request using this JSON \" \\\n        \"format params string. This is helpful for unit tests and debugging. \" \\\n        \"When specified jobID must NOT be specified. [default: %default].\")\n\n  parser.add_option(\"--clearModels\", action=\"store_true\", default=False,\n        help=\"clear out the models table before starting [default: %default].\")\n\n  parser.add_option(\"--resetJobStatus\", action=\"store_true\", default=False,\n        help=\"Reset the job status before starting  [default: %default].\")\n\n  parser.add_option(\"--logLevel\", action=\"store\", type=\"int\", default=None,\n        help=\"override default log level. Pass in an integer value that \"\n        \"represents the desired logging level (10=logging.DEBUG, \"\n        \"20=logging.INFO, etc.) [default: %default].\")\n\n  # Evaluate command line arguments\n  (options, args) = parser.parse_args(argv[1:])\n  if len(args) != 0:\n    raise RuntimeError(\"Expected no command line arguments, but got: %s\" % \\\n                        (args))\n\n  if (options.jobID and options.params):\n    raise RuntimeError(\"--jobID and --params can not be used at the same time\")\n\n  if (options.jobID is None and options.params is None):\n    raise RuntimeError(\"Either --jobID or --params must be specified.\")\n\n  initLogging(verbose=True)\n\n  # Instantiate the HypersearchWorker and run it\n  hst = HypersearchWorker(options, argv[1:])\n\n  # Normal use. This is one of among a number of workers. If we encounter\n  #  an exception at the outer loop here, we fail the entire job.\n  if options.params is None:\n    try:\n      jobID = hst.run()\n\n    except Exception, e:\n      jobID = options.jobID\n      msg = StringIO.StringIO()\n      print >>msg, \"%s: Exception occurred in Hypersearch Worker: %r\" % \\\n         (ErrorCodes.hypersearchLogicErr, e)\n      traceback.print_exc(None, msg)\n\n      completionReason = ClientJobsDAO.CMPL_REASON_ERROR\n      completionMsg = msg.getvalue()\n      hst.logger.error(completionMsg)\n\n      # If no other worker already marked the job as failed, do so now.\n      jobsDAO = ClientJobsDAO.get()\n      workerCmpReason = jobsDAO.jobGetFields(options.jobID,\n          ['workerCompletionReason'])[0]\n      if workerCmpReason == ClientJobsDAO.CMPL_REASON_SUCCESS:\n        jobsDAO.jobSetFields(options.jobID, fields=dict(\n            cancel=True,\n            workerCompletionReason = ClientJobsDAO.CMPL_REASON_ERROR,\n            workerCompletionMsg = completionMsg),\n            useConnectionID=False,\n            ignoreUnchanged=True)\n\n\n  # Run just 1 worker for the entire job. Used for unit tests that run in\n  # 1 process\n  else:\n    jobID = None\n    completionReason = ClientJobsDAO.CMPL_REASON_SUCCESS\n    completionMsg = \"Success\"\n\n    try:\n      jobID = hst.run()\n    except Exception, e:\n      jobID = hst._options.jobID\n      completionReason = ClientJobsDAO.CMPL_REASON_ERROR\n      completionMsg = \"ERROR: %s\" % (e,)\n      raise\n    finally:\n      if jobID is not None:\n        cjDAO = ClientJobsDAO.get()\n        cjDAO.jobSetCompleted(jobID=jobID,\n                              completionReason=completionReason,\n                              completionMsg=completionMsg)\n\n  return jobID", "response": "This is the main function of the HypersearchWorker script. It is used by unit test code\n                when calling this function."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrunning this worker. Parameters: ---------------------------------------------------------------------- retval: jobID of the job we ran. This is used by unit test code when calling this working using the --params command line option (which tells this worker to insert the job itself).", "response": "def run(self):\n    \"\"\" Run this worker.\n\n    Parameters:\n    ----------------------------------------------------------------------\n    retval:     jobID of the job we ran. This is used by unit test code\n                  when calling this working using the --params command\n                  line option (which tells this worker to insert the job\n                  itself).\n    \"\"\"\n    # Easier access to options\n    options = self._options\n\n    # ---------------------------------------------------------------------\n    # Connect to the jobs database\n    self.logger.info(\"Connecting to the jobs database\")\n    cjDAO = ClientJobsDAO.get()\n\n    # Get our worker ID\n    self._workerID = cjDAO.getConnectionID()\n\n    if options.clearModels:\n      cjDAO.modelsClearAll()\n\n    # -------------------------------------------------------------------------\n    # if params were specified on the command line, insert a new job using\n    #  them.\n    if options.params is not None:\n      options.jobID = cjDAO.jobInsert(client='hwTest', cmdLine=\"echo 'test mode'\",\n                  params=options.params, alreadyRunning=True,\n                  minimumWorkers=1, maximumWorkers=1,\n                  jobType = cjDAO.JOB_TYPE_HS)\n    if options.workerID is not None:\n      wID = options.workerID\n    else:\n      wID = self._workerID\n\n    buildID = Configuration.get('nupic.software.buildNumber', 'N/A')\n    logPrefix = '<BUILDID=%s, WORKER=HW, WRKID=%s, JOBID=%s> ' % \\\n                (buildID, wID, options.jobID)\n    ExtendedLogger.setLogPrefix(logPrefix)\n\n    # ---------------------------------------------------------------------\n    # Get the search parameters\n    # If asked to reset the job status, do that now\n    if options.resetJobStatus:\n      cjDAO.jobSetFields(options.jobID,\n           fields={'workerCompletionReason': ClientJobsDAO.CMPL_REASON_SUCCESS,\n                   'cancel': False,\n                   #'engWorkerState': None\n                   },\n           useConnectionID=False,\n           ignoreUnchanged=True)\n    jobInfo = cjDAO.jobInfo(options.jobID)\n    self.logger.info(\"Job info retrieved: %s\" % (str(clippedObj(jobInfo))))\n\n\n    # ---------------------------------------------------------------------\n    # Instantiate the Hypersearch object, which will handle the logic of\n    #  which models to create when we need more to evaluate.\n    jobParams = json.loads(jobInfo.params)\n\n    # Validate job params\n    jsonSchemaPath = os.path.join(os.path.dirname(__file__),\n                                  \"jsonschema\",\n                                  \"jobParamsSchema.json\")\n    validate(jobParams, schemaPath=jsonSchemaPath)\n\n\n    hsVersion = jobParams.get('hsVersion', None)\n    if hsVersion == 'v2':\n      self._hs = HypersearchV2(searchParams=jobParams, workerID=self._workerID,\n              cjDAO=cjDAO, jobID=options.jobID, logLevel=options.logLevel)\n    else:\n      raise RuntimeError(\"Invalid Hypersearch implementation (%s) specified\" \\\n                          % (hsVersion))\n\n\n    # =====================================================================\n    # The main loop.\n    try:\n      exit = False\n      numModelsTotal = 0\n      print >>sys.stderr, \"reporter:status:Evaluating first model...\"\n      while not exit:\n\n        # ------------------------------------------------------------------\n        # Choose a model to evaluate\n        batchSize = 10              # How many to try at a time.\n        modelIDToRun = None\n        while modelIDToRun is None:\n\n          if options.modelID is None:\n            # -----------------------------------------------------------------\n            # Get the latest results on all running models and send them to\n            #  the Hypersearch implementation\n            # This calls cjDAO.modelsGetUpdateCounters(), compares the\n            # updateCounters with what we have cached, fetches the results for the\n            # changed and new models, and sends those to the Hypersearch\n            # implementation's self._hs.recordModelProgress() method.\n            self._processUpdatedModels(cjDAO)\n\n            # --------------------------------------------------------------------\n            # Create a new batch of models\n            (exit, newModels) = self._hs.createModels(numModels = batchSize)\n            if exit:\n              break\n\n            # No more models left to create, just loop. The _hs is waiting for\n            #   all remaining running models to complete, and may pick up on an\n            #  orphan if it detects one.\n            if len(newModels) == 0:\n              continue\n\n            # Try and insert one that we will run\n            for (modelParams, modelParamsHash, particleHash) in newModels:\n              jsonModelParams = json.dumps(modelParams)\n              (modelID, ours) = cjDAO.modelInsertAndStart(options.jobID,\n                                  jsonModelParams, modelParamsHash, particleHash)\n\n              # Some other worker is already running it, tell the Hypersearch object\n              #  so that it doesn't try and insert it again\n              if not ours:\n                mParamsAndHash = cjDAO.modelsGetParams([modelID])[0]\n                mResult = cjDAO.modelsGetResultAndStatus([modelID])[0]\n                results = mResult.results\n                if results is not None:\n                  results = json.loads(results)\n\n                modelParams = json.loads(mParamsAndHash.params)\n                particleHash = cjDAO.modelsGetFields(modelID,\n                                  ['engParticleHash'])[0]\n                particleInst = \"%s.%s\" % (\n                          modelParams['particleState']['id'],\n                          modelParams['particleState']['genIdx'])\n                self.logger.info(\"Adding model %d to our internal DB \" \\\n                      \"because modelInsertAndStart() failed to insert it: \" \\\n                      \"paramsHash=%s, particleHash=%s, particleId='%s'\", modelID,\n                      mParamsAndHash.engParamsHash.encode('hex'),\n                      particleHash.encode('hex'), particleInst)\n                self._hs.recordModelProgress(modelID = modelID,\n                      modelParams = modelParams,\n                      modelParamsHash = mParamsAndHash.engParamsHash,\n                      results = results,\n                      completed = (mResult.status == cjDAO.STATUS_COMPLETED),\n                      completionReason = mResult.completionReason,\n                      matured = mResult.engMatured,\n                      numRecords = mResult.numRecords)\n              else:\n                modelIDToRun = modelID\n                break\n\n          else:\n            # A specific modelID was passed on the command line\n            modelIDToRun = int(options.modelID)\n            mParamsAndHash = cjDAO.modelsGetParams([modelIDToRun])[0]\n            modelParams = json.loads(mParamsAndHash.params)\n            modelParamsHash = mParamsAndHash.engParamsHash\n\n            # Make us the worker\n            cjDAO.modelSetFields(modelIDToRun,\n                                     dict(engWorkerConnId=self._workerID))\n            if False:\n              # Change the hash and params of the old entry so that we can\n              #  create a new model with the same params\n              for attempt in range(1000):\n                paramsHash = hashlib.md5(\"OrphanParams.%d.%d\" % (modelIDToRun,\n                                                                 attempt)).digest()\n                particleHash = hashlib.md5(\"OrphanParticle.%d.%d\" % (modelIDToRun,\n                                                                  attempt)).digest()\n                try:\n                  cjDAO.modelSetFields(modelIDToRun,\n                                           dict(engParamsHash=paramsHash,\n                                                engParticleHash=particleHash))\n                  success = True\n                except:\n                  success = False\n                if success:\n                  break\n              if not success:\n                raise RuntimeError(\"Unexpected failure to change paramsHash and \"\n                                   \"particleHash of orphaned model\")\n\n              (modelIDToRun, ours) = cjDAO.modelInsertAndStart(options.jobID,\n                                  mParamsAndHash.params, modelParamsHash)\n\n\n\n            # ^^^ end while modelIDToRun ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n        # ---------------------------------------------------------------\n        # We have a model, evaluate it now\n        # All done?\n        if exit:\n          break\n\n        # Run the model now\n        self.logger.info(\"RUNNING MODEL GID=%d, paramsHash=%s, params=%s\",\n              modelIDToRun, modelParamsHash.encode('hex'), modelParams)\n\n        # ---------------------------------------------------------------------\n        # Construct model checkpoint GUID for this model:\n        # jobParams['persistentJobGUID'] contains the client's (e.g., API Server)\n        # persistent, globally-unique model identifier, which is what we need;\n        persistentJobGUID = jobParams['persistentJobGUID']\n        assert persistentJobGUID, \"persistentJobGUID: %r\" % (persistentJobGUID,)\n\n        modelCheckpointGUID = jobInfo.client + \"_\" + persistentJobGUID + (\n          '_' + str(modelIDToRun))\n\n\n        self._hs.runModel(modelID=modelIDToRun, jobID = options.jobID,\n                          modelParams=modelParams, modelParamsHash=modelParamsHash,\n                          jobsDAO=cjDAO, modelCheckpointGUID=modelCheckpointGUID)\n\n        # TODO: don't increment for orphaned models\n        numModelsTotal += 1\n\n        self.logger.info(\"COMPLETED MODEL GID=%d; EVALUATED %d MODELs\",\n          modelIDToRun, numModelsTotal)\n        print >>sys.stderr, \"reporter:status:Evaluated %d models...\" % \\\n                                    (numModelsTotal)\n        print >>sys.stderr, \"reporter:counter:HypersearchWorker,numModels,1\"\n\n        if options.modelID is not None:\n          exit = True\n        # ^^^ end while not exit\n\n    finally:\n      # Provide Hypersearch instance an opportunity to clean up temporary files\n      self._hs.close()\n\n    self.logger.info(\"FINISHED. Evaluated %d models.\" % (numModelsTotal))\n    print >>sys.stderr, \"reporter:status:Finished, evaluated %d models\" % (numModelsTotal)\n    return options.jobID"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getBucketIndices(self, x):\n\n    if ((isinstance(x, float) and math.isnan(x)) or\n        x == SENTINEL_VALUE_FOR_MISSING_DATA):\n      return [None]\n\n    if self._offset is None:\n      self._offset = x\n\n    bucketIdx = (\n        (self._maxBuckets/2) + int(round((x - self._offset) / self.resolution))\n    )\n\n    if bucketIdx < 0:\n      bucketIdx = 0\n    elif bucketIdx >= self._maxBuckets:\n      bucketIdx = self._maxBuckets-1\n\n    return [bucketIdx]", "response": "Returns the indices of the bucket that contains x."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef mapBucketIndexToNonZeroBits(self, index):\n    if index < 0:\n      index = 0\n\n    if index >= self._maxBuckets:\n      index = self._maxBuckets-1\n\n    if not self.bucketMap.has_key(index):\n      if self.verbosity >= 2:\n        print \"Adding additional buckets to handle index=\", index\n      self._createBucket(index)\n    return self.bucketMap[index]", "response": "Given a bucket index return the list of non - zero bits."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nencode x into a base. py array.", "response": "def encodeIntoArray(self, x, output):\n    \"\"\" See method description in base.py \"\"\"\n\n    if x is not None and not isinstance(x, numbers.Number):\n      raise TypeError(\n          \"Expected a scalar input but got input of type %s\" % type(x))\n\n    # Get the bucket index to use\n    bucketIdx = self.getBucketIndices(x)[0]\n\n    # None is returned for missing value in which case we return all 0's.\n    output[0:self.n] = 0\n    if bucketIdx is not None:\n      output[self.mapBucketIndexToNonZeroBits(bucketIdx)] = 1"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _createBucket(self, index):\n    if index < self.minIndex:\n      if index == self.minIndex - 1:\n        # Create a new representation that has exactly w-1 overlapping bits\n        # as the min representation\n        self.bucketMap[index] = self._newRepresentation(self.minIndex,\n                                                        index)\n        self.minIndex = index\n      else:\n        # Recursively create all the indices above and then this index\n        self._createBucket(index+1)\n        self._createBucket(index)\n    else:\n      if index == self.maxIndex + 1:\n        # Create a new representation that has exactly w-1 overlapping bits\n        # as the max representation\n        self.bucketMap[index] = self._newRepresentation(self.maxIndex,\n                                                        index)\n        self.maxIndex = index\n      else:\n        # Recursively create all the indices below and then this index\n        self._createBucket(index-1)\n        self._createBucket(index)", "response": "Create the given bucket index. Recursively create as many in - between\n    bucket indices as necessary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _newRepresentation(self, index, newIndex):\n    newRepresentation = self.bucketMap[index].copy()\n\n    # Choose the bit we will replace in this representation. We need to shift\n    # this bit deterministically. If this is always chosen randomly then there\n    # is a 1 in w chance of the same bit being replaced in neighboring\n    # representations, which is fairly high\n    ri = newIndex % self.w\n\n    # Now we choose a bit such that the overlap rules are satisfied.\n    newBit = self.random.getUInt32(self.n)\n    newRepresentation[ri] = newBit\n    while newBit in self.bucketMap[index] or \\\n          not self._newRepresentationOK(newRepresentation, newIndex):\n      self.numTries += 1\n      newBit = self.random.getUInt32(self.n)\n      newRepresentation[ri] = newBit\n\n    return newRepresentation", "response": "Return a new representation for newIndex that overlaps with the\n    representation at index."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns True if this new candidate representation satisfies all our overlap rules.", "response": "def _newRepresentationOK(self, newRep, newIndex):\n    \"\"\"\n    Return True if this new candidate representation satisfies all our overlap\n    rules. Since we know that neighboring representations differ by at most\n    one bit, we compute running overlaps.\n    \"\"\"\n    if newRep.size != self.w:\n      return False\n    if (newIndex < self.minIndex-1) or (newIndex > self.maxIndex+1):\n      raise ValueError(\"newIndex must be within one of existing indices\")\n\n    # A binary representation of newRep. We will use this to test containment\n    newRepBinary = numpy.array([False]*self.n)\n    newRepBinary[newRep] = True\n\n    # Midpoint\n    midIdx = self._maxBuckets/2\n\n    # Start by checking the overlap at minIndex\n    runningOverlap = self._countOverlap(self.bucketMap[self.minIndex], newRep)\n    if not self._overlapOK(self.minIndex, newIndex, overlap=runningOverlap):\n      return False\n\n    # Compute running overlaps all the way to the midpoint\n    for i in range(self.minIndex+1, midIdx+1):\n      # This is the bit that is going to change\n      newBit = (i-1)%self.w\n\n      # Update our running overlap\n      if newRepBinary[self.bucketMap[i-1][newBit]]:\n        runningOverlap -= 1\n      if newRepBinary[self.bucketMap[i][newBit]]:\n        runningOverlap += 1\n\n      # Verify our rules\n      if not self._overlapOK(i, newIndex, overlap=runningOverlap):\n        return False\n\n    # At this point, runningOverlap contains the overlap for midIdx\n    # Compute running overlaps all the way to maxIndex\n    for i in range(midIdx+1, self.maxIndex+1):\n      # This is the bit that is going to change\n      newBit = i%self.w\n\n      # Update our running overlap\n      if newRepBinary[self.bucketMap[i-1][newBit]]:\n        runningOverlap -= 1\n      if newRepBinary[self.bucketMap[i][newBit]]:\n        runningOverlap += 1\n\n      # Verify our rules\n      if not self._overlapOK(i, newIndex, overlap=runningOverlap):\n        return False\n\n    return True"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the number of overlap between bucket indices i and j", "response": "def _countOverlapIndices(self, i, j):\n    \"\"\"\n    Return the overlap between bucket indices i and j\n    \"\"\"\n    if self.bucketMap.has_key(i) and self.bucketMap.has_key(j):\n      iRep = self.bucketMap[i]\n      jRep = self.bucketMap[j]\n      return self._countOverlap(iRep, jRep)\n    else:\n      raise ValueError(\"Either i or j don't exist\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncounts the number of overlap between two representations.", "response": "def _countOverlap(rep1, rep2):\n    \"\"\"\n    Return the overlap between two representations. rep1 and rep2 are lists of\n    non-zero indices.\n    \"\"\"\n    overlap = 0\n    for e in rep1:\n      if e in rep2:\n        overlap += 1\n    return overlap"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _overlapOK(self, i, j, overlap=None):\n    if overlap is None:\n      overlap = self._countOverlapIndices(i, j)\n    if abs(i-j) < self.w:\n      if overlap == (self.w - abs(i-j)):\n        return True\n      else:\n        return False\n    else:\n      if overlap <= self._maxOverlap:\n        return True\n      else:\n        return False", "response": "Returns True if the given overlap between bucket indices i and j are acceptable."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ninitialize the bucket map assuming the given number of maxBuckets.", "response": "def _initializeBucketMap(self, maxBuckets, offset):\n    \"\"\"\n    Initialize the bucket map assuming the given number of maxBuckets.\n    \"\"\"\n    # The first bucket index will be _maxBuckets / 2 and bucket indices will be\n    # allowed to grow lower or higher as long as they don't become negative.\n    # _maxBuckets is required because the current SDR Classifier assumes bucket\n    # indices must be non-negative. This normally does not need to be changed\n    # but if altered, should be set to an even number.\n    self._maxBuckets = maxBuckets\n    self.minIndex = self._maxBuckets / 2\n    self.maxIndex = self._maxBuckets / 2\n\n    # The scalar offset used to map scalar values to bucket indices. The middle\n    # bucket will correspond to numbers in the range\n    # [offset-resolution/2, offset+resolution/2).\n    # The bucket index for a number x will be:\n    #     maxBuckets/2 + int( round( (x-offset)/resolution ) )\n    self._offset = offset\n\n    # This dictionary maps a bucket index into its bit representation\n    # We initialize the class with a single bucket with index 0\n    self.bucketMap = {}\n\n    def _permutation(n):\n      r = numpy.arange(n, dtype=numpy.uint32)\n      self.random.shuffle(r)\n      return r\n\n    self.bucketMap[self.minIndex] = _permutation(self.n)[0:self.w]\n\n    # How often we need to retry when generating valid encodings\n    self.numTries = 0"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a decorator that returns a closure suitable for use as a decorator for retrySQL.", "response": "def retrySQL(timeoutSec=60*5, logger=None):\n  \"\"\" Return a closure suitable for use as a decorator for\n  retrying a pymysql DAO function on certain failures that warrant retries (\n  e.g., RDS/MySQL server down temporarily, transaction deadlock, etc.).\n  We share this function across multiple scripts (e.g., ClientJobsDAO,\n  StreamMgr) for consitent behavior.\n\n  .. note:: Please ensure that the operation being retried is idempotent.\n\n  .. note:: logging must be initialized *before* any loggers are created, else\n     there will be no output; see nupic.support.initLogging()\n\n  Usage Example:\n\n  .. code-block:: python\n\n    @retrySQL()\n    def jobInfo(self, jobID):\n        ...\n\n  :param timeoutSec:       How many seconds from time of initial call to stop retrying\n                     (floating point)\n  :param logger:           User-supplied logger instance.\n\n  \"\"\"\n\n  if logger is None:\n    logger = logging.getLogger(__name__)\n\n  def retryFilter(e, args, kwargs):\n\n    if isinstance(e, (pymysql.InternalError, pymysql.OperationalError)):\n      if e.args and e.args[0] in _ALL_RETRIABLE_ERROR_CODES:\n        return True\n\n    elif isinstance(e, pymysql.Error):\n      if (e.args and\n          inspect.isclass(e.args[0]) and issubclass(e.args[0], socket_error)):\n        return True\n\n    return False\n\n\n  retryExceptions = tuple([\n    pymysql.InternalError,\n    pymysql.OperationalError,\n    pymysql.Error,\n  ])\n\n  return make_retry_decorator(\n    timeoutSec=timeoutSec, initialRetryDelaySec=0.1, maxRetryDelaySec=10,\n    retryExceptions=retryExceptions, retryFilter=retryFilter,\n    logger=logger)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create(*args, **kwargs):\n    impl = kwargs.pop('implementation', None)\n    if impl is None:\n      impl = Configuration.get('nupic.opf.sdrClassifier.implementation')\n    if impl == 'py':\n      return SDRClassifier(*args, **kwargs)\n    elif impl == 'cpp':\n      return FastSDRClassifier(*args, **kwargs)\n    elif impl == 'diff':\n      return SDRClassifierDiff(*args, **kwargs)\n    else:\n      raise ValueError('Invalid classifier implementation (%r). Value must be '\n                       '\"py\", \"cpp\" or \"diff\".' % impl)", "response": "Create a new SDR classifier factory."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread the object containing the next set of elements in the sequence.", "response": "def read(proto):\n    \"\"\"\n    :param proto: SDRClassifierRegionProto capnproto object\n    \"\"\"\n    impl = proto.implementation\n    if impl == 'py':\n      return SDRClassifier.read(proto.sdrClassifier)\n    elif impl == 'cpp':\n      return FastSDRClassifier.read(proto.sdrClassifier)\n    elif impl == 'diff':\n      return SDRClassifierDiff.read(proto.sdrClassifier)\n    else:\n      raise ValueError('Invalid classifier implementation (%r). Value must be '\n                       '\"py\", \"cpp\" or \"diff\".' % impl)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncross list of sequences.", "response": "def cross_list(*sequences):\n  \"\"\"\n  From: http://book.opensourceproject.org.cn/lamp/python/pythoncook2/opensource/0596007973/pythoncook2-chp-19-sect-9.html\n  \"\"\"\n  result = [[ ]]\n  for seq in sequences:\n    result = [sublist+[item] for sublist in result for item in seq]\n  return result"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncrossing - product the given sequence.", "response": "def cross(*sequences):\n  \"\"\"\n  From: http://book.opensourceproject.org.cn/lamp/python/pythoncook2/opensource/0596007973/pythoncook2-chp-19-sect-9.html\n  \"\"\"\n  # visualize an odometer, with \"wheels\" displaying \"digits\"...:\n  wheels = map(iter, sequences)\n  digits = [it.next( ) for it in wheels]\n  while True:\n    yield tuple(digits)\n    for i in range(len(digits)-1, -1, -1):\n      try:\n        digits[i] = wheels[i].next( )\n        break\n      except StopIteration:\n        wheels[i] = iter(sequences[i])\n        digits[i] = wheels[i].next( )\n    else:\n      break"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef dcross(**keywords):\n  keys = keywords.keys()\n  # Could use keywords.values(), but unsure whether the order\n  # the values come out in is guaranteed to be the same as that of keys\n  # (appears to be anecdotally true).\n  sequences = [keywords[key] for key in keys]\n\n  wheels = map(iter, sequences)\n  digits = [it.next( ) for it in wheels]\n  while True:\n    yield dict(zip(keys, digits))\n    for i in range(len(digits)-1, -1, -1):\n      try:\n        digits[i] = wheels[i].next( )\n        break\n      except StopIteration:\n        wheels[i] = iter(sequences[i])\n        digits[i] = wheels[i].next( )\n    else:\n      break", "response": "Similar to cross but generates output dictionaries instead of tuples."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef mmGetMetricFromTrace(self, trace):\n    return Metric.createFromTrace(trace.makeCountsTrace(),\n                                  excludeResets=self.mmGetTraceResets())", "response": "Returns a metric over an indices trace excluding resets."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a metric that counts the number of predicted active cells per column for each sequence.", "response": "def mmGetMetricSequencesPredictedActiveCellsPerColumn(self):\n    \"\"\"\n    Metric for number of predicted => active cells per column for each sequence\n\n    @return (Metric) metric\n    \"\"\"\n    self._mmComputeTransitionTraces()\n\n    numCellsPerColumn = []\n\n    for predictedActiveCells in (\n        self._mmData[\"predictedActiveCellsForSequence\"].values()):\n      cellsForColumn = self.mapCellsToColumns(predictedActiveCells)\n      numCellsPerColumn += [len(x) for x in cellsForColumn.values()]\n\n    return Metric(self,\n                  \"# predicted => active cells per column for each sequence\",\n                  numCellsPerColumn)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a metric that counts how many sequences each predicted active cell appears in.", "response": "def mmGetMetricSequencesPredictedActiveCellsShared(self):\n    \"\"\"\n    Metric for number of sequences each predicted => active cell appears in\n\n    Note: This metric is flawed when it comes to high-order sequences.\n\n    @return (Metric) metric\n    \"\"\"\n    self._mmComputeTransitionTraces()\n\n    numSequencesForCell = defaultdict(lambda: 0)\n\n    for predictedActiveCells in (\n          self._mmData[\"predictedActiveCellsForSequence\"].values()):\n      for cell in predictedActiveCells:\n        numSequencesForCell[cell] += 1\n\n    return Metric(self,\n                  \"# sequences each predicted => active cells appears in\",\n                  numSequencesForCell.values())"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef mmPrettyPrintConnections(self):\n    text = \"\"\n\n    text += (\"Segments: (format => \"\n             \"(#) [(source cell=permanence ...),       ...]\\n\")\n    text += \"------------------------------------\\n\"\n\n    columns = range(self.numberOfColumns())\n\n    for column in columns:\n      cells = self.cellsForColumn(column)\n\n      for cell in cells:\n        segmentDict = dict()\n\n        for seg in self.connections.segmentsForCell(cell):\n          synapseList = []\n\n          for synapse in self.connections.synapsesForSegment(seg):\n            synapseData = self.connections.dataForSynapse(synapse)\n            synapseList.append(\n                (synapseData.presynapticCell, synapseData.permanence))\n\n          synapseList.sort()\n          synapseStringList = [\"{0:3}={1:.2f}\".format(sourceCell, permanence) for\n                               sourceCell, permanence in synapseList]\n          segmentDict[seg] = \"({0})\".format(\" \".join(synapseStringList))\n\n        text += (\"Column {0:3} / Cell {1:3}:\\t({2}) {3}\\n\".format(\n          column, cell,\n          len(segmentDict.values()),\n          \"[{0}]\".format(\",       \".join(segmentDict.values()))))\n\n      if column < len(columns) - 1:  # not last\n        text += \"\\n\"\n\n    text += \"------------------------------------\\n\"\n\n    return text", "response": "Pretty print the connections in the temporal memory."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef createTemporalAnomaly(recordParams, spatialParams=_SP_PARAMS,\n                          temporalParams=_TM_PARAMS,\n                          verbosity=_VERBOSITY):\n\n\n  \"\"\"Generates a Network with connected RecordSensor, SP, TM.\n\n  This function takes care of generating regions and the canonical links.\n  The network has a sensor region reading data from a specified input and\n  passing the encoded representation to an SPRegion.\n  The SPRegion output is passed to a TMRegion.\n\n  Note: this function returns a network that needs to be initialized. This\n  allows the user to extend the network by adding further regions and\n  connections.\n\n  :param recordParams: a dict with parameters for creating RecordSensor region.\n  :param spatialParams: a dict with parameters for creating SPRegion.\n  :param temporalParams: a dict with parameters for creating TMRegion.\n  :param verbosity: an integer representing how chatty the network will be.\n  \"\"\"\n  inputFilePath = recordParams[\"inputFilePath\"]\n  scalarEncoderArgs = recordParams[\"scalarEncoderArgs\"]\n  dateEncoderArgs = recordParams[\"dateEncoderArgs\"]\n\n  scalarEncoder = ScalarEncoder(**scalarEncoderArgs)\n  dateEncoder = DateEncoder(**dateEncoderArgs)\n\n  encoder = MultiEncoder()\n  encoder.addEncoder(scalarEncoderArgs[\"name\"], scalarEncoder)\n  encoder.addEncoder(dateEncoderArgs[\"name\"], dateEncoder)\n\n  network = Network()\n\n  network.addRegion(\"sensor\", \"py.RecordSensor\",\n                    json.dumps({\"verbosity\": verbosity}))\n\n  sensor = network.regions[\"sensor\"].getSelf()\n  sensor.encoder = encoder\n  sensor.dataSource = FileRecordStream(streamID=inputFilePath)\n\n  # Create the spatial pooler region\n  spatialParams[\"inputWidth\"] = sensor.encoder.getWidth()\n  network.addRegion(\"spatialPoolerRegion\", \"py.SPRegion\",\n                    json.dumps(spatialParams))\n\n  # Link the SP region to the sensor input\n  network.link(\"sensor\", \"spatialPoolerRegion\", \"UniformLink\", \"\")\n  network.link(\"sensor\", \"spatialPoolerRegion\", \"UniformLink\", \"\",\n               srcOutput=\"resetOut\", destInput=\"resetIn\")\n  network.link(\"spatialPoolerRegion\", \"sensor\", \"UniformLink\", \"\",\n               srcOutput=\"spatialTopDownOut\", destInput=\"spatialTopDownIn\")\n  network.link(\"spatialPoolerRegion\", \"sensor\", \"UniformLink\", \"\",\n               srcOutput=\"temporalTopDownOut\", destInput=\"temporalTopDownIn\")\n\n  # Add the TPRegion on top of the SPRegion\n  network.addRegion(\"temporalPoolerRegion\", \"py.TMRegion\",\n                    json.dumps(temporalParams))\n\n  network.link(\"spatialPoolerRegion\", \"temporalPoolerRegion\", \"UniformLink\", \"\")\n  network.link(\"temporalPoolerRegion\", \"spatialPoolerRegion\", \"UniformLink\", \"\",\n               srcOutput=\"topDownOut\", destInput=\"topDownIn\")\n\n  spatialPoolerRegion = network.regions[\"spatialPoolerRegion\"]\n\n  # Make sure learning is enabled\n  spatialPoolerRegion.setParameter(\"learningMode\", True)\n  # We want temporal anomalies so disable anomalyMode in the SP. This mode is\n  # used for computing anomalies in a non-temporal model.\n  spatialPoolerRegion.setParameter(\"anomalyMode\", False)\n\n  temporalPoolerRegion = network.regions[\"temporalPoolerRegion\"]\n\n  # Enable topDownMode to get the predicted columns output\n  temporalPoolerRegion.setParameter(\"topDownMode\", True)\n  # Make sure learning is enabled (this is the default)\n  temporalPoolerRegion.setParameter(\"learningMode\", True)\n  # Enable inference mode so we get predictions\n  temporalPoolerRegion.setParameter(\"inferenceMode\", True)\n  # Enable anomalyMode to compute the anomaly score.\n  temporalPoolerRegion.setParameter(\"anomalyMode\", True)\n\n  return network", "response": "This function creates a Network with connected RecordSensor and a temporal anomaly."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nruns the network and write output to writer. Run the network and write output to writer.", "response": "def runNetwork(network, writer):\n  \"\"\"Run the network and write output to writer.\n\n  :param network: a Network instance to run\n  :param writer: a csv.writer instance to write output to\n  \"\"\"\n  sensorRegion = network.regions[\"sensor\"]\n  temporalPoolerRegion = network.regions[\"temporalPoolerRegion\"]\n\n  for i in xrange(_NUM_RECORDS):\n    # Run the network for a single iteration\n    network.run(1)\n\n    # Write out the anomaly score along with the record number and consumption\n    # value.\n    anomalyScore = temporalPoolerRegion.getOutputData(\"anomalyScore\")[0]\n    consumption = sensorRegion.getOutputData(\"sourceOut\")[0]\n    writer.writerow((i, consumption, anomalyScore))"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nappends the given list of PeriodicActivityRequest elements to the internal list of Activations.", "response": "def __appendActivities(self, periodicActivities):\n    \"\"\"\n    periodicActivities: A sequence of PeriodicActivityRequest elements\n    \"\"\"\n\n    for req in periodicActivities:\n      act =   self.Activity(repeating=req.repeating,\n                            period=req.period,\n                            cb=req.cb,\n                            iteratorHolder=[iter(xrange(req.period-1))])\n      self.__activities.append(act)\n\n    return"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd a value over a range of rows.", "response": "def add(reader, writer, column, start, stop, value):\n  \"\"\"Adds a value over a range of rows.\n\n  Args:\n    reader: A FileRecordStream object with input data.\n    writer: A FileRecordStream object to write output data to.\n    column: The column of data to modify.\n    start: The first row in the range to modify.\n    end: The last row in the range to modify.\n    value: The value to add.\n  \"\"\"\n  for i, row in enumerate(reader):\n    if i >= start and i <= stop:\n      row[column] = type(value)(row[column]) + value\n    writer.appendRecord(row)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef scale(reader, writer, column, start, stop, multiple):\n  for i, row in enumerate(reader):\n    if i >= start and i <= stop:\n      row[column] = type(multiple)(row[column]) * multiple\n    writer.appendRecord(row)", "response": "Multiply a value over a range of rows."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncopies a range of values to a new location in the data set.", "response": "def copy(reader, writer, start, stop, insertLocation=None, tsCol=None):\n  \"\"\"Copies a range of values to a new location in the data set.\n\n  Args:\n    reader: A FileRecordStream object with input data.\n    writer: A FileRecordStream object to write output data to.\n    start: The first row in the range to copy.\n    stop: The last row in the range to copy.\n    insertLocation: The location to insert the copied range. If not specified,\n        the range is inserted immediately following itself.\n  \"\"\"\n  assert stop >= start\n  startRows = []\n  copyRows = []\n  ts = None\n  inc = None\n  if tsCol is None:\n    tsCol = reader.getTimestampFieldIdx()\n  for i, row in enumerate(reader):\n    # Get the first timestamp and the increment.\n    if ts is None:\n      ts = row[tsCol]\n    elif inc is None:\n      inc = row[tsCol] - ts\n    # Keep a list of all rows and a list of rows to copy.\n    if i >= start and i <= stop:\n      copyRows.append(row)\n    startRows.append(row)\n  # Insert the copied rows.\n  if insertLocation is None:\n    insertLocation = stop + 1\n  startRows[insertLocation:insertLocation] = copyRows\n  # Update the timestamps.\n  for row in startRows:\n    row[tsCol] = ts\n    writer.appendRecord(row)\n    ts += inc"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsampling n rows from the specified range.", "response": "def sample(reader, writer, n, start=None, stop=None, tsCol=None,\n           writeSampleOnly=True):\n  \"\"\"Samples n rows.\n\n  Args:\n    reader: A FileRecordStream object with input data.\n    writer: A FileRecordStream object to write output data to.\n    n: The number of elements to sample.\n    start: The first row in the range to sample from.\n    stop: The last row in the range to sample from.\n    tsCol: If specified, the timestamp column to update.\n    writeSampleOnly: If False, the rows before start are written before the\n        sample and the rows after stop are written after the sample.\n  \"\"\"\n  rows = list(reader)\n  if tsCol is not None:\n    ts = rows[0][tsCol]\n    inc = rows[1][tsCol] - ts\n  if start is None:\n    start = 0\n  if stop is None:\n    stop = len(rows) - 1\n  initialN = stop - start + 1\n  # Select random rows in the sample range to delete until the desired number\n  # of rows are left.\n  numDeletes =  initialN - n\n  for i in xrange(numDeletes):\n    delIndex = random.randint(start, stop - i)\n    del rows[delIndex]\n  # Remove outside rows if specified.\n  if writeSampleOnly:\n    rows = rows[start:start + n]\n  # Rewrite columns if tsCol is given.\n  if tsCol is not None:\n    ts = rows[0][tsCol]\n  # Write resulting rows.\n  for row in rows:\n    if tsCol is not None:\n      row[tsCol] = ts\n      ts += inc\n    writer.appendRecord(row)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize the encoder for the current version of the class.", "response": "def _initEncoder(self, w, minval, maxval, n, radius, resolution):\n    \"\"\" (helper function)  There are three different ways of thinking about the representation.\n     Handle each case here.\"\"\"\n    if n != 0:\n      if (radius !=0 or resolution != 0):\n        raise ValueError(\"Only one of n/radius/resolution can be specified for a ScalarEncoder\")\n      assert n > w\n      self.n = n\n\n      if (minval is not None and maxval is not None):\n        if not self.periodic:\n          self.resolution = float(self.rangeInternal) / (self.n - self.w)\n        else:\n          self.resolution = float(self.rangeInternal) / (self.n)\n\n        self.radius = self.w * self.resolution\n\n        if self.periodic:\n          self.range = self.rangeInternal\n        else:\n          self.range = self.rangeInternal + self.resolution\n\n    else:\n      if radius != 0:\n        if (resolution != 0):\n          raise ValueError(\"Only one of radius/resolution can be specified for a ScalarEncoder\")\n        self.radius = radius\n        self.resolution = float(self.radius) / w\n      elif resolution != 0:\n        self.resolution = float(resolution)\n        self.radius = self.resolution * self.w\n      else:\n        raise Exception(\"One of n, radius, resolution must be specified for a ScalarEncoder\")\n\n      if (minval is not None and maxval is not None):\n        if self.periodic:\n          self.range = self.rangeInternal\n        else:\n          self.range = self.rangeInternal + self.resolution\n\n        nfloat = self.w * (self.range / self.radius) + 2 * self.padding\n        self.n = int(math.ceil(nfloat))"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the bit offset of the first bit to be set in the encoder output. For periodic encoders, this can be a negative number when the encoded output wraps around.", "response": "def _getFirstOnBit(self, input):\n    \"\"\" Return the bit offset of the first bit to be set in the encoder output.\n    For periodic encoders, this can be a negative number when the encoded output\n    wraps around. \"\"\"\n\n    if input == SENTINEL_VALUE_FOR_MISSING_DATA:\n      return [None]\n\n    else:\n      if input < self.minval:\n        # Don't clip periodic inputs. Out-of-range input is always an error\n        if self.clipInput and not self.periodic:\n          if self.verbosity > 0:\n            print \"Clipped input %s=%.2f to minval %.2f\" % (self.name, input,\n                                                            self.minval)\n          input = self.minval\n        else:\n          raise Exception('input (%s) less than range (%s - %s)' %\n                          (str(input), str(self.minval), str(self.maxval)))\n\n      if self.periodic:\n        # Don't clip periodic inputs. Out-of-range input is always an error\n        if input >= self.maxval:\n          raise Exception('input (%s) greater than periodic range (%s - %s)' %\n                          (str(input), str(self.minval), str(self.maxval)))\n      else:\n        if input > self.maxval:\n          if self.clipInput:\n            if self.verbosity > 0:\n              print \"Clipped input %s=%.2f to maxval %.2f\" % (self.name, input,\n                                                              self.maxval)\n            input = self.maxval\n          else:\n            raise Exception('input (%s) greater than range (%s - %s)' %\n                            (str(input), str(self.minval), str(self.maxval)))\n\n      if self.periodic:\n        centerbin = int((input - self.minval) * self.nInternal / self.range) \\\n                      + self.padding\n      else:\n        centerbin = int(((input - self.minval) + self.resolution/2) \\\n                          / self.resolution ) + self.padding\n\n\n      # We use the first bit to be set in the encoded output as the bucket index\n      minbin = centerbin - self.halfwidth\n      return [minbin]"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getBucketIndices(self, input):\n\n    if type(input) is float and math.isnan(input):\n      input = SENTINEL_VALUE_FOR_MISSING_DATA\n\n    if input == SENTINEL_VALUE_FOR_MISSING_DATA:\n      return [None]\n\n    minbin = self._getFirstOnBit(input)[0]\n\n    # For periodic encoders, the bucket index is the index of the center bit\n    if self.periodic:\n      bucketIdx = minbin + self.halfwidth\n      if bucketIdx < 0:\n        bucketIdx += self.n\n\n    # for non-periodic encoders, the bucket index is the index of the left bit\n    else:\n      bucketIdx = minbin\n\n    return [bucketIdx]", "response": "Returns the bucket indices of the input bit."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef encodeIntoArray(self, input, output, learn=True):\n\n    if input is not None and not isinstance(input, numbers.Number):\n      raise TypeError(\n          \"Expected a scalar input but got input of type %s\" % type(input))\n\n    if type(input) is float and math.isnan(input):\n      input = SENTINEL_VALUE_FOR_MISSING_DATA\n\n    # Get the bucket index to use\n    bucketIdx = self._getFirstOnBit(input)[0]\n\n    if bucketIdx is None:\n      # None is returned for missing value\n      output[0:self.n] = 0  #TODO: should all 1s, or random SDR be returned instead?\n\n    else:\n      # The bucket index is the index of the first bit to set in the output\n      output[:self.n] = 0\n      minbin = bucketIdx\n      maxbin = minbin + 2*self.halfwidth\n      if self.periodic:\n        # Handle the edges by computing wrap-around\n        if maxbin >= self.n:\n          bottombins = maxbin - self.n + 1\n          output[:bottombins] = 1\n          maxbin = self.n - 1\n        if minbin < 0:\n          topbins = -minbin\n          output[self.n - topbins:self.n] = 1\n          minbin = 0\n\n      assert minbin >= 0\n      assert maxbin < self.n\n      # set the output (except for periodic wraparound)\n      output[minbin:maxbin + 1] = 1\n\n    # Debug the decode() method\n    if self.verbosity >= 2:\n      print\n      print \"input:\", input\n      print \"range:\", self.minval, \"-\", self.maxval\n      print \"n:\", self.n, \"w:\", self.w, \"resolution:\", self.resolution, \\\n            \"radius\", self.radius, \"periodic:\", self.periodic\n      print \"output:\",\n      self.pprint(output)\n      print \"input desc:\", self.decodedToStr(self.decode(output))", "response": "Encode the input into an array of integers."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsee the function description in base.py", "response": "def decode(self, encoded, parentFieldName=''):\n    \"\"\" See the function description in base.py\n    \"\"\"\n\n    # For now, we simply assume any top-down output greater than 0\n    #  is ON. Eventually, we will probably want to incorporate the strength\n    #  of each top-down output.\n    tmpOutput = numpy.array(encoded[:self.n] > 0).astype(encoded.dtype)\n    if not tmpOutput.any():\n      return (dict(), [])\n\n    # ------------------------------------------------------------------------\n    # First, assume the input pool is not sampled 100%, and fill in the\n    #  \"holes\" in the encoded representation (which are likely to be present\n    #  if this is a coincidence that was learned by the SP).\n\n    # Search for portions of the output that have \"holes\"\n    maxZerosInARow = self.halfwidth\n    for i in xrange(maxZerosInARow):\n      searchStr = numpy.ones(i + 3, dtype=encoded.dtype)\n      searchStr[1:-1] = 0\n      subLen = len(searchStr)\n\n      # Does this search string appear in the output?\n      if self.periodic:\n        for j in xrange(self.n):\n          outputIndices = numpy.arange(j, j + subLen)\n          outputIndices %= self.n\n          if numpy.array_equal(searchStr, tmpOutput[outputIndices]):\n            tmpOutput[outputIndices] = 1\n\n      else:\n        for j in xrange(self.n - subLen + 1):\n          if numpy.array_equal(searchStr, tmpOutput[j:j + subLen]):\n            tmpOutput[j:j + subLen] = 1\n\n\n    if self.verbosity >= 2:\n      print \"raw output:\", encoded[:self.n]\n      print \"filtered output:\", tmpOutput\n\n    # ------------------------------------------------------------------------\n    # Find each run of 1's.\n    nz = tmpOutput.nonzero()[0]\n    runs = []     # will be tuples of (startIdx, runLength)\n    run = [nz[0], 1]\n    i = 1\n    while (i < len(nz)):\n      if nz[i] == run[0] + run[1]:\n        run[1] += 1\n      else:\n        runs.append(run)\n        run = [nz[i], 1]\n      i += 1\n    runs.append(run)\n\n    # If we have a periodic encoder, merge the first and last run if they\n    #  both go all the way to the edges\n    if self.periodic and len(runs) > 1:\n      if runs[0][0] == 0 and runs[-1][0] + runs[-1][1] == self.n:\n        runs[-1][1] += runs[0][1]\n        runs = runs[1:]\n\n\n    # ------------------------------------------------------------------------\n    # Now, for each group of 1's, determine the \"left\" and \"right\" edges, where\n    #  the \"left\" edge is inset by halfwidth and the \"right\" edge is inset by\n    #  halfwidth.\n    # For a group of width w or less, the \"left\" and \"right\" edge are both at\n    #   the center position of the group.\n    ranges = []\n    for run in runs:\n      (start, runLen) = run\n      if runLen <= self.w:\n        left = right = start + runLen / 2\n      else:\n        left = start + self.halfwidth\n        right = start + runLen - 1 - self.halfwidth\n\n      # Convert to input space.\n      if not self.periodic:\n        inMin = (left - self.padding) * self.resolution + self.minval\n        inMax = (right - self.padding) * self.resolution + self.minval\n      else:\n        inMin = (left - self.padding) * self.range / self.nInternal + self.minval\n        inMax = (right - self.padding) * self.range / self.nInternal + self.minval\n      # Handle wrap-around if periodic\n      if self.periodic:\n        if inMin >= self.maxval:\n          inMin -= self.range\n          inMax -= self.range\n\n      # Clip low end\n      if inMin < self.minval:\n        inMin = self.minval\n      if inMax < self.minval:\n        inMax = self.minval\n\n      # If we have a periodic encoder, and the max is past the edge, break into\n      #  2 separate ranges\n      if self.periodic and inMax >= self.maxval:\n        ranges.append([inMin, self.maxval])\n        ranges.append([self.minval, inMax - self.range])\n      else:\n        if inMax > self.maxval:\n          inMax = self.maxval\n        if inMin > self.maxval:\n          inMin = self.maxval\n        ranges.append([inMin, inMax])\n\n    desc = self._generateRangeDescription(ranges)\n    # Return result\n    if parentFieldName != '':\n      fieldName = \"%s.%s\" % (parentFieldName, self.name)\n    else:\n      fieldName = self.name\n\n    return ({fieldName: (ranges, desc)}, [fieldName])"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _generateRangeDescription(self, ranges):\n    desc = \"\"\n    numRanges = len(ranges)\n    for i in xrange(numRanges):\n      if ranges[i][0] != ranges[i][1]:\n        desc += \"%.2f-%.2f\" % (ranges[i][0], ranges[i][1])\n      else:\n        desc += \"%.2f\" % (ranges[i][0])\n      if i < numRanges - 1:\n        desc += \", \"\n    return desc", "response": "generate description from a text description of the ranges"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the interal _topDownMappingM matrix used for handling the bucketInfo and topDownCompute methods. This matrix is used for handling the bucketInfo and topDownCompute methods. This matrix is used for handling the bucketInfo and topDownCompute methods. This matrix is used for handling the bucketInfo and topDownCompute methods.", "response": "def _getTopDownMapping(self):\n    \"\"\" Return the interal _topDownMappingM matrix used for handling the\n    bucketInfo() and topDownCompute() methods. This is a matrix, one row per\n    category (bucket) where each row contains the encoded output for that\n    category.\n    \"\"\"\n\n    # Do we need to build up our reverse mapping table?\n    if self._topDownMappingM is None:\n\n      # The input scalar value corresponding to each possible output encoding\n      if self.periodic:\n        self._topDownValues = numpy.arange(self.minval + self.resolution / 2.0,\n                                           self.maxval,\n                                           self.resolution)\n      else:\n        #Number of values is (max-min)/resolutions\n        self._topDownValues = numpy.arange(self.minval,\n                                           self.maxval + self.resolution / 2.0,\n                                           self.resolution)\n\n      # Each row represents an encoded output pattern\n      numCategories = len(self._topDownValues)\n      self._topDownMappingM = SM32(numCategories, self.n)\n\n      outputSpace = numpy.zeros(self.n, dtype=GetNTAReal())\n      for i in xrange(numCategories):\n        value = self._topDownValues[i]\n        value = max(value, self.minval)\n        value = min(value, self.maxval)\n        self.encodeIntoArray(value, outputSpace, learn=False)\n        self._topDownMappingM.setRowFromDense(i, outputSpace)\n\n    return self._topDownMappingM"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getBucketValues(self):\n\n    # Need to re-create?\n    if self._bucketValues is None:\n      topDownMappingM = self._getTopDownMapping()\n      numBuckets = topDownMappingM.nRows()\n      self._bucketValues = []\n      for bucketIdx in range(numBuckets):\n        self._bucketValues.append(self.getBucketInfo([bucketIdx])[0].value)\n\n    return self._bucketValues", "response": "Returns a list of all the bucket values in the current project."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef getBucketInfo(self, buckets):\n\n    # Get/generate the topDown mapping table\n    #NOTE: although variable topDownMappingM is unused, some (bad-style) actions\n    #are executed during _getTopDownMapping() so this line must stay here\n    topDownMappingM = self._getTopDownMapping()\n\n    # The \"category\" is simply the bucket index\n    category = buckets[0]\n    encoding = self._topDownMappingM.getRow(category)\n\n    # Which input value does this correspond to?\n    if self.periodic:\n      inputVal = (self.minval + (self.resolution / 2.0) +\n                  (category * self.resolution))\n    else:\n      inputVal = self.minval + (category * self.resolution)\n\n    return [EncoderResult(value=inputVal, scalar=inputVal, encoding=encoding)]", "response": "Returns the encoder result for the given buckets."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef topDownCompute(self, encoded):\n\n    # Get/generate the topDown mapping table\n    topDownMappingM = self._getTopDownMapping()\n\n    # See which \"category\" we match the closest.\n    category = topDownMappingM.rightVecProd(encoded).argmax()\n\n    # Return that bucket info\n    return self.getBucketInfo([category])", "response": "Compute the topDown mapping table for a given encoded word."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef closenessScores(self, expValues, actValues, fractional=True):\n\n    expValue = expValues[0]\n    actValue = actValues[0]\n    if self.periodic:\n      expValue = expValue % self.maxval\n      actValue = actValue % self.maxval\n\n    err = abs(expValue - actValue)\n    if self.periodic:\n      err = min(err, self.maxval - err)\n    if fractional:\n      pctErr = float(err) / (self.maxval - self.minval)\n      pctErr = min(1.0, pctErr)\n      closeness = 1.0 - pctErr\n    else:\n      closeness = err\n\n    return numpy.array([closeness])", "response": "Compute closeness scores for the given expValues and actValues."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninitializes all ephemeral members after being restored to a pickled state.", "response": "def _initEphemerals(self):\n    \"\"\"\n    Initialize all ephemeral members after being restored to a pickled state.\n    \"\"\"\n    ## We store the lists of segments updates, per cell, so that they can be\n    # applied later during learning, when the cell gets bottom-up activation.\n    # We store one list per cell. The lists are identified with a hash key which\n    # is a tuple (column index, cell index).\n    self.segmentUpdates = {}\n\n    # Allocate and reset all stats\n    self.resetStats()\n\n    # NOTE: We don't use the same backtrack buffer for inference and learning\n    # because learning has a different metric for determining if an input from\n    # the past is potentially useful again for backtracking.\n    #\n    # Our inference backtrack buffer. This keeps track of up to\n    # maxInfBacktrack of previous input. Each entry is a list of active column\n    # inputs.\n    self._prevInfPatterns = []\n\n    # Our learning backtrack buffer. This keeps track of up to maxLrnBacktrack\n    # of previous input. Each entry is a list of active column inputs\n    self._prevLrnPatterns = []\n\n    # Keep integers rather than bools. Float?\n    stateShape = (self.numberOfCols, self.cellsPerColumn)\n\n    self.lrnActiveState = {}\n    self.lrnActiveState[\"t\"] = numpy.zeros(stateShape, dtype=\"int8\")\n    self.lrnActiveState[\"t-1\"] = numpy.zeros(stateShape, dtype=\"int8\")\n\n    self.lrnPredictedState = {}\n    self.lrnPredictedState[\"t\"] = numpy.zeros(stateShape, dtype=\"int8\")\n    self.lrnPredictedState[\"t-1\"] = numpy.zeros(stateShape, dtype=\"int8\")\n\n    self.infActiveState = {}\n    self.infActiveState[\"t\"] = numpy.zeros(stateShape, dtype=\"int8\")\n    self.infActiveState[\"t-1\"] = numpy.zeros(stateShape, dtype=\"int8\")\n    self.infActiveState[\"backup\"] = numpy.zeros(stateShape, dtype=\"int8\")\n    self.infActiveState[\"candidate\"] = numpy.zeros(stateShape, dtype=\"int8\")\n\n    self.infPredictedState = {}\n    self.infPredictedState[\"t\"] = numpy.zeros(stateShape, dtype=\"int8\")\n    self.infPredictedState[\"t-1\"] = numpy.zeros(stateShape, dtype=\"int8\")\n    self.infPredictedState[\"backup\"] = numpy.zeros(stateShape, dtype=\"int8\")\n    self.infPredictedState[\"candidate\"] = numpy.zeros(stateShape, dtype=\"int8\")\n\n    self.cellConfidence = {}\n    self.cellConfidence[\"t\"] = numpy.zeros(stateShape, dtype=\"float32\")\n    self.cellConfidence[\"t-1\"] = numpy.zeros(stateShape, dtype=\"float32\")\n    self.cellConfidence[\"candidate\"] = numpy.zeros(stateShape, dtype=\"float32\")\n\n    self.colConfidence = {}\n    self.colConfidence[\"t\"] = numpy.zeros(self.numberOfCols, dtype=\"float32\")\n    self.colConfidence[\"t-1\"] = numpy.zeros(self.numberOfCols, dtype=\"float32\")\n    self.colConfidence[\"candidate\"] = numpy.zeros(self.numberOfCols,\n                                                  dtype=\"float32\")"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef write(self, proto):\n    proto.version = TM_VERSION\n    self._random.write(proto.random)\n    proto.numberOfCols = self.numberOfCols\n    proto.cellsPerColumn = self.cellsPerColumn\n    proto.initialPerm = float(self.initialPerm)\n    proto.connectedPerm = float(self.connectedPerm)\n    proto.minThreshold = self.minThreshold\n    proto.newSynapseCount = self.newSynapseCount\n    proto.permanenceInc = float(self.permanenceInc)\n    proto.permanenceDec = float(self.permanenceDec)\n    proto.permanenceMax = float(self.permanenceMax)\n    proto.globalDecay = float(self.globalDecay)\n    proto.activationThreshold = self.activationThreshold\n    proto.doPooling = self.doPooling\n    proto.segUpdateValidDuration = self.segUpdateValidDuration\n    proto.burnIn = self.burnIn\n    proto.collectStats = self.collectStats\n    proto.verbosity = self.verbosity\n    proto.pamLength = self.pamLength\n    proto.maxAge = self.maxAge\n    proto.maxInfBacktrack = self.maxInfBacktrack\n    proto.maxLrnBacktrack = self.maxLrnBacktrack\n    proto.maxSeqLength = self.maxSeqLength\n    proto.maxSegmentsPerCell = self.maxSegmentsPerCell\n    proto.maxSynapsesPerSegment = self.maxSynapsesPerSegment\n    proto.outputType = self.outputType\n\n    proto.activeColumns = self.activeColumns\n\n    cellListProto = proto.init(\"cells\", len(self.cells))\n    for i, columnSegments in enumerate(self.cells):\n      columnSegmentsProto = cellListProto.init(i, len(columnSegments))\n      for j, cellSegments in enumerate(columnSegments):\n        cellSegmentsProto = columnSegmentsProto.init(j, len(cellSegments))\n        for k, segment in enumerate(cellSegments):\n          segment.write(cellSegmentsProto[k])\n\n    proto.lrnIterationIdx = self.lrnIterationIdx\n    proto.iterationIdx = self.iterationIdx\n    proto.segID = self.segID\n    if self.currentOutput is None:\n      proto.currentOutput.none = None\n    else:\n      proto.currentOutput.list = self.currentOutput.tolist()\n    proto.pamCounter = self.pamCounter\n    proto.collectSequenceStats = self.collectSequenceStats\n    proto.resetCalled = self.resetCalled\n    # In case of None, use negative value as placeholder for serialization\n    proto.avgInputDensity = self.avgInputDensity or -1.0\n    proto.learnedSeqLength = self.learnedSeqLength\n    proto.avgLearnedSeqLength = self.avgLearnedSeqLength\n\n    proto.prevLrnPatterns = self._prevLrnPatterns\n    proto.prevInfPatterns = self._prevInfPatterns\n\n    segmentUpdatesListProto = proto.init(\"segmentUpdates\",\n                                         len(self.segmentUpdates))\n    for i, (key, updates) in enumerate(self.segmentUpdates.iteritems()):\n      cellSegmentUpdatesProto = segmentUpdatesListProto[i]\n      cellSegmentUpdatesProto.columnIdx = key[0]\n      cellSegmentUpdatesProto.cellIdx = key[1]\n      segmentUpdatesProto = cellSegmentUpdatesProto.init(\"segmentUpdates\",\n                                                         len(updates))\n      for j, (lrnIterationIdx, segmentUpdate) in enumerate(updates):\n        segmentUpdateWrapperProto = segmentUpdatesProto[j]\n        segmentUpdateWrapperProto.lrnIterationIdx = lrnIterationIdx\n        segmentUpdate.write(segmentUpdateWrapperProto.segmentUpdate)\n\n    # self.cellConfidence\n    proto.cellConfidenceT = self.cellConfidence[\"t\"].tolist()\n    proto.cellConfidenceT1 = self.cellConfidence[\"t-1\"].tolist()\n    proto.cellConfidenceCandidate = self.cellConfidence[\"candidate\"].tolist()\n\n    # self.colConfidence\n    proto.colConfidenceT = self.colConfidence[\"t\"].tolist()\n    proto.colConfidenceT1 = self.colConfidence[\"t-1\"].tolist()\n    proto.colConfidenceCandidate = self.colConfidence[\"candidate\"].tolist()\n\n    # self.lrnActiveState\n    proto.lrnActiveStateT = self.lrnActiveState[\"t\"].tolist()\n    proto.lrnActiveStateT1 = self.lrnActiveState[\"t-1\"].tolist()\n\n    # self.infActiveState\n    proto.infActiveStateT = self.infActiveState[\"t\"].tolist()\n    proto.infActiveStateT1 = self.infActiveState[\"t-1\"].tolist()\n    proto.infActiveStateBackup = self.infActiveState[\"backup\"].tolist()\n    proto.infActiveStateCandidate = self.infActiveState[\"candidate\"].tolist()\n\n    # self.lrnPredictedState\n    proto.lrnPredictedStateT = self.lrnPredictedState[\"t\"].tolist()\n    proto.lrnPredictedStateT1 = self.lrnPredictedState[\"t-1\"].tolist()\n\n    # self.infPredictedState\n    proto.infPredictedStateT = self.infPredictedState[\"t\"].tolist()\n    proto.infPredictedStateT1 = self.infPredictedState[\"t-1\"].tolist()\n    proto.infPredictedStateBackup = self.infPredictedState[\"backup\"].tolist()\n    proto.infPredictedStateCandidate = self.infPredictedState[\"candidate\"].tolist()\n\n    proto.consolePrinterVerbosity = self.consolePrinterVerbosity", "response": "Populates the contents of the given proto instance."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nresetting the internal state of all cells and resets the internal state of all cells.", "response": "def reset(self,):\n    \"\"\"\n    Reset the state of all cells.\n\n    This is normally used between sequences while training. All internal states\n    are reset to 0.\n    \"\"\"\n    if self.verbosity >= 3:\n      print \"\\n==== RESET =====\"\n\n    self.lrnActiveState['t-1'].fill(0)\n    self.lrnActiveState['t'].fill(0)\n    self.lrnPredictedState['t-1'].fill(0)\n    self.lrnPredictedState['t'].fill(0)\n\n    self.infActiveState['t-1'].fill(0)\n    self.infActiveState['t'].fill(0)\n    self.infPredictedState['t-1'].fill(0)\n    self.infPredictedState['t'].fill(0)\n\n    self.cellConfidence['t-1'].fill(0)\n    self.cellConfidence['t'].fill(0)\n\n    # Flush the segment update queue\n    self.segmentUpdates = {}\n\n    self._internalStats['nInfersSinceReset'] = 0\n\n    #To be removed\n    self._internalStats['curPredictionScore'] = 0\n    #New prediction score\n    self._internalStats['curPredictionScore2']   = 0\n    self._internalStats['curFalseNegativeScore'] = 0\n    self._internalStats['curFalsePositiveScore'] = 0\n\n    self._internalStats['curMissing'] = 0\n    self._internalStats['curExtra'] = 0\n\n    # When a reset occurs, set prevSequenceSignature to the signature of the\n    # just-completed sequence and start accumulating histogram for the next\n    # sequence.\n    self._internalStats['prevSequenceSignature'] = None\n    if self.collectSequenceStats:\n      if self._internalStats['confHistogram'].sum() > 0:\n        sig = self._internalStats['confHistogram'].copy()\n        sig.reshape(self.numberOfCols * self.cellsPerColumn)\n        self._internalStats['prevSequenceSignature'] = sig\n      self._internalStats['confHistogram'].fill(0)\n\n    self.resetCalled = True\n\n    # Clear out input history\n    self._prevInfPatterns = []\n    self._prevLrnPatterns = []"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _updateStatsInferEnd(self, stats, bottomUpNZ, predictedState,\n                           colConfidence):\n    \"\"\"\n    Called at the end of learning and inference, this routine will update\n    a number of stats in our _internalStats dictionary, including our computed\n    prediction score.\n\n    :param stats            internal stats dictionary\n    :param bottomUpNZ       list of the active bottom-up inputs\n    :param predictedState   The columns we predicted on the last time step (should\n                            match the current bottomUpNZ in the best case)\n    :param colConfidence    Column confidences we determined on the last time step\n    \"\"\"\n    # Return if not collecting stats\n    if not self.collectStats:\n      return\n    stats['nInfersSinceReset'] += 1\n\n    # Compute the prediction score, how well the prediction from the last\n    # time step predicted the current bottom-up input\n    (numExtra2, numMissing2, confidences2) = self._checkPrediction(\n        patternNZs=[bottomUpNZ], output=predictedState,\n        colConfidence=colConfidence)\n    predictionScore, positivePredictionScore, negativePredictionScore = (\n        confidences2[0])\n\n    # Store the stats that don't depend on burn-in\n    stats['curPredictionScore2'] = float(predictionScore)\n    stats['curFalseNegativeScore'] = 1.0 - float(positivePredictionScore)\n    stats['curFalsePositiveScore'] = float(negativePredictionScore)\n\n    stats['curMissing'] = numMissing2\n    stats['curExtra'] = numExtra2\n\n    # If we are passed the burn-in period, update the accumulated stats\n    # Here's what various burn-in values mean:\n    #   0: try to predict the first element of each sequence and all subsequent\n    #   1: try to predict the second element of each sequence and all subsequent\n    #   etc.\n    if stats['nInfersSinceReset'] <= self.burnIn:\n      return\n\n    # Burn-in related stats\n    stats['nPredictions'] += 1\n    numExpected = max(1.0, float(len(bottomUpNZ)))\n\n    stats['totalMissing'] += numMissing2\n    stats['totalExtra'] += numExtra2\n    stats['pctExtraTotal'] += 100.0 * numExtra2 / numExpected\n    stats['pctMissingTotal'] += 100.0 * numMissing2 / numExpected\n    stats['predictionScoreTotal2'] += float(predictionScore)\n    stats['falseNegativeScoreTotal'] += 1.0 - float(positivePredictionScore)\n    stats['falsePositiveScoreTotal'] += float(negativePredictionScore)\n\n    if self.collectSequenceStats:\n      # Collect cell confidences for every cell that correctly predicted current\n      # bottom up input. Normalize confidence across each column\n      cc = self.cellConfidence['t-1'] * self.infActiveState['t']\n      sconf = cc.sum(axis=1)\n      for c in range(self.numberOfCols):\n        if sconf[c] > 0:\n          cc[c, :] /= sconf[c]\n\n      # Update cell confidence histogram: add column-normalized confidence\n      # scores to the histogram\n      self._internalStats['confHistogram'] += cc", "response": "This routine is called at the end of learning and inference of a single active bottom - up input. It updates the internal stats dictionary with the number of stats that depend on the current bottom - up input."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprint an integer array that is the same shape as activeState.", "response": "def printState(self, aState):\n    \"\"\"\n    Print an integer array that is the same shape as activeState.\n\n    :param aState: TODO: document\n    \"\"\"\n    def formatRow(var, i):\n      s = ''\n      for c in range(self.numberOfCols):\n        if c > 0 and c % 10 == 0:\n          s += ' '\n        s += str(var[c, i])\n      s += ' '\n      return s\n\n    for i in xrange(self.cellsPerColumn):\n      print formatRow(aState, i)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef printConfidence(self, aState, maxCols = 20):\n    def formatFPRow(var, i):\n      s = ''\n      for c in range(min(maxCols, self.numberOfCols)):\n        if c > 0 and c % 10 == 0:\n          s += '   '\n        s += ' %5.3f' % var[c, i]\n      s += ' '\n      return s\n\n    for i in xrange(self.cellsPerColumn):\n      print formatFPRow(aState, i)", "response": "Print a floating point array that is the same shape as activeState."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef printColConfidence(self, aState, maxCols = 20):\n    def formatFPRow(var):\n      s = ''\n      for c in range(min(maxCols, self.numberOfCols)):\n        if c > 0 and c % 10 == 0:\n          s += '   '\n        s += ' %5.3f' % var[c]\n      s += ' '\n      return s\n\n    print formatFPRow(aState)", "response": "Print the col confidence of a flat floating point array."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nprints the current state of the current language.", "response": "def printStates(self, printPrevious = True, printLearnState = True):\n    \"\"\"\n    TODO: document\n    \n    :param printPrevious: \n    :param printLearnState: \n    :return: \n    \"\"\"\n    def formatRow(var, i):\n      s = ''\n      for c in range(self.numberOfCols):\n        if c > 0 and c % 10 == 0:\n          s += ' '\n        s += str(var[c, i])\n      s += ' '\n      return s\n\n    print \"\\nInference Active state\"\n    for i in xrange(self.cellsPerColumn):\n      if printPrevious:\n        print formatRow(self.infActiveState['t-1'], i),\n      print formatRow(self.infActiveState['t'], i)\n\n    print \"Inference Predicted state\"\n    for i in xrange(self.cellsPerColumn):\n      if printPrevious:\n        print formatRow(self.infPredictedState['t-1'], i),\n      print formatRow(self.infPredictedState['t'], i)\n\n    if printLearnState:\n      print \"\\nLearn Active state\"\n      for i in xrange(self.cellsPerColumn):\n        if printPrevious:\n          print formatRow(self.lrnActiveState['t-1'], i),\n        print formatRow(self.lrnActiveState['t'], i)\n\n      print \"Learn Predicted state\"\n      for i in xrange(self.cellsPerColumn):\n        if printPrevious:\n          print formatRow(self.lrnPredictedState['t-1'], i),\n        print formatRow(self.lrnPredictedState['t'], i)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef printOutput(self, y):\n    print \"Output\"\n    for i in xrange(self.cellsPerColumn):\n      for c in xrange(self.numberOfCols):\n        print int(y[c, i]),\n      print", "response": "Print the output of the sequence of documents."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef printInput(self, x):\n    print \"Input\"\n    for c in xrange(self.numberOfCols):\n      print int(x[c]),\n    print", "response": "Print the input of the sequence."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef printParameters(self):\n    print \"numberOfCols=\", self.numberOfCols\n    print \"cellsPerColumn=\", self.cellsPerColumn\n    print \"minThreshold=\", self.minThreshold\n    print \"newSynapseCount=\", self.newSynapseCount\n    print \"activationThreshold=\", self.activationThreshold\n    print\n    print \"initialPerm=\", self.initialPerm\n    print \"connectedPerm=\", self.connectedPerm\n    print \"permanenceInc=\", self.permanenceInc\n    print \"permanenceDec=\", self.permanenceDec\n    print \"permanenceMax=\", self.permanenceMax\n    print \"globalDecay=\", self.globalDecay\n    print\n    print \"doPooling=\", self.doPooling\n    print \"segUpdateValidDuration=\", self.segUpdateValidDuration\n    print \"pamLength=\", self.pamLength", "response": "Prints the parameters for the TM."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nprint the list of column cellIdxs for each of the active cells in state.", "response": "def printActiveIndices(self, state, andValues=False):\n    \"\"\"\n    Print the list of ``[column, cellIdx]`` indices for each of the active cells \n    in state.\n\n    :param state: TODO: document\n    :param andValues: TODO: document\n    \"\"\"\n    if len(state.shape) == 2:\n      (cols, cellIdxs) = state.nonzero()\n    else:\n      cols = state.nonzero()[0]\n      cellIdxs = numpy.zeros(len(cols))\n\n    if len(cols) == 0:\n      print \"NONE\"\n      return\n\n    prevCol = -1\n    for (col, cellIdx) in zip(cols, cellIdxs):\n      if col != prevCol:\n        if prevCol != -1:\n          print \"] \",\n        print \"Col %d: [\" % (col),\n        prevCol = col\n\n      if andValues:\n        if len(state.shape) == 2:\n          value = state[col, cellIdx]\n        else:\n          value = state[col]\n        print \"%d: %s,\" % (cellIdx, value),\n      else:\n        print \"%d,\" % (cellIdx),\n    print \"]\""}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef printComputeEnd(self, output, learn=False):\n    if self.verbosity >= 3:\n      print \"----- computeEnd summary: \"\n      print \"learn:\", learn\n      print \"numBurstingCols: %s, \" % (\n          self.infActiveState['t'].min(axis=1).sum()),\n      print \"curPredScore2: %s, \" % (\n          self._internalStats['curPredictionScore2']),\n      print \"curFalsePosScore: %s, \" % (\n          self._internalStats['curFalsePositiveScore']),\n      print \"1-curFalseNegScore: %s, \" % (\n          1 - self._internalStats['curFalseNegativeScore'])\n      print \"numSegments: \", self.getNumSegments(),\n      print \"avgLearnedSeqLength: \", self.avgLearnedSeqLength\n\n      print \"----- infActiveState (%d on) ------\" % (\n          self.infActiveState['t'].sum())\n      self.printActiveIndices(self.infActiveState['t'])\n      if self.verbosity >= 6:\n        self.printState(self.infActiveState['t'])\n\n      print \"----- infPredictedState (%d on)-----\" % (\n          self.infPredictedState['t'].sum())\n      self.printActiveIndices(self.infPredictedState['t'])\n      if self.verbosity >= 6:\n        self.printState(self.infPredictedState['t'])\n\n      print \"----- lrnActiveState (%d on) ------\" % (\n          self.lrnActiveState['t'].sum())\n      self.printActiveIndices(self.lrnActiveState['t'])\n      if self.verbosity >= 6:\n        self.printState(self.lrnActiveState['t'])\n\n      print \"----- lrnPredictedState (%d on)-----\" % (\n          self.lrnPredictedState['t'].sum())\n      self.printActiveIndices(self.lrnPredictedState['t'])\n      if self.verbosity >= 6:\n        self.printState(self.lrnPredictedState['t'])\n\n\n      print \"----- cellConfidence -----\"\n      self.printActiveIndices(self.cellConfidence['t'], andValues=True)\n      if self.verbosity >= 6:\n        self.printConfidence(self.cellConfidence['t'])\n\n      print \"----- colConfidence -----\"\n      self.printActiveIndices(self.colConfidence['t'], andValues=True)\n\n      print \"----- cellConfidence[t-1] for currently active cells -----\"\n      cc = self.cellConfidence['t-1'] * self.infActiveState['t']\n      self.printActiveIndices(cc, andValues=True)\n\n      if self.verbosity == 4:\n        print \"Cells, predicted segments only:\"\n        self.printCells(predictedOnly=True)\n      elif self.verbosity >= 5:\n        print \"Cells, all segments:\"\n        self.printCells(predictedOnly=False)\n      print\n\n    elif self.verbosity >= 1:\n      print \"TM: learn:\", learn\n      print \"TM: active outputs(%d):\" % len(output.nonzero()[0]),\n      self.printActiveIndices(output.reshape(self.numberOfCols,\n                                             self.cellsPerColumn))", "response": "Prints out the diagnostic for the current language."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nprints the cell with the given index.", "response": "def printCell(self, c, i, onlyActiveSegments=False):\n    \"\"\"\n    TODO: document\n    \n    :param c: \n    :param i: \n    :param onlyActiveSegments: \n    :return: \n    \"\"\"\n\n    if len(self.cells[c][i]) > 0:\n      print \"Column\", c, \"Cell\", i, \":\",\n      print len(self.cells[c][i]), \"segment(s)\"\n      for j, s in enumerate(self.cells[c][i]):\n        isActive = self._isSegmentActive(s, self.infActiveState['t'])\n        if not onlyActiveSegments or isActive:\n          isActiveStr = \"*\" if isActive else \" \"\n          print \"  %sSeg #%-3d\" % (isActiveStr, j),\n          s.debugPrint()"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef printCells(self, predictedOnly=False):\n    if predictedOnly:\n      print \"--- PREDICTED CELLS ---\"\n    else:\n      print \"--- ALL CELLS ---\"\n    print \"Activation threshold=\", self.activationThreshold,\n    print \"min threshold=\", self.minThreshold,\n    print \"connected perm=\", self.connectedPerm\n\n    for c in xrange(self.numberOfCols):\n      for i in xrange(self.cellsPerColumn):\n        if not predictedOnly or self.infPredictedState['t'][c, i]:\n          self.printCell(c, i, predictedOnly)", "response": "Prints the cells of the current node."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getSegmentOnCell(self, c, i, segIdx):\n    seg = self.cells[c][i][segIdx]\n    retlist = [[seg.segID, seg.isSequenceSeg, seg.positiveActivations,\n                seg.totalActivations, seg.lastActiveIteration,\n                seg._lastPosDutyCycle, seg._lastPosDutyCycleIteration]]\n    retlist += seg.syns\n    return retlist", "response": "returns a list of segments representing the segment on the cell i"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _addToSegmentUpdates(self, c, i, segUpdate):\n    # Sometimes we might be passed an empty update\n    if segUpdate is None or len(segUpdate.activeSynapses) == 0:\n      return\n\n    key = (c, i) # key = (column index, cell index in column)\n\n    # TODO: scan list of updates for that cell and consolidate?\n    # But watch out for dates!\n    if self.segmentUpdates.has_key(key):\n      self.segmentUpdates[key] += [(self.lrnIterationIdx, segUpdate)]\n    else:\n      self.segmentUpdates[key] = [(self.lrnIterationIdx, segUpdate)]", "response": "Add a new segment update to the list of segment updates."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _computeOutput(self):\n    # TODO: This operation can be sped up by:\n    #  1.)  Pre-allocating space for the currentOutput\n    #  2.)  Making predictedState and activeState of type 'float32' up front\n    #  3.)  Using logical_or(self.predictedState['t'], self.activeState['t'],\n    #          self.currentOutput)\n\n    if self.outputType == 'activeState1CellPerCol':\n\n      # Fire only the most confident cell in columns that have 2 or more\n      #  active cells\n      mostActiveCellPerCol = self.cellConfidence['t'].argmax(axis=1)\n      self.currentOutput = numpy.zeros(self.infActiveState['t'].shape,\n                                       dtype='float32')\n\n      # Turn on the most confident cell in each column. Note here that\n      #  Columns refers to TM columns, even though each TM column is a row\n      #  in the numpy array.\n      numCols = self.currentOutput.shape[0]\n      self.currentOutput[(xrange(numCols), mostActiveCellPerCol)] = 1\n\n      # Don't turn on anything in columns which are not active at all\n      activeCols = self.infActiveState['t'].max(axis=1)\n      inactiveCols = numpy.where(activeCols==0)[0]\n      self.currentOutput[inactiveCols, :] = 0\n\n\n    elif self.outputType == 'activeState':\n      self.currentOutput = self.infActiveState['t']\n\n    elif self.outputType == 'normal':\n      self.currentOutput = numpy.logical_or(self.infPredictedState['t'],\n                                            self.infActiveState['t'])\n\n    else:\n      raise RuntimeError(\"Unimplemented outputType\")\n\n    return self.currentOutput.reshape(-1).astype('float32')", "response": "Compute the output for both learning and inference."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef predict(self, nSteps):\n    # Save the TM dynamic state, we will use to revert back in the end\n    pristineTPDynamicState = self._getTPDynamicState()\n\n    assert (nSteps>0)\n\n    # multiStepColumnPredictions holds all the future prediction.\n    multiStepColumnPredictions = numpy.zeros((nSteps, self.numberOfCols),\n                                             dtype=\"float32\")\n\n    # This is a (nSteps-1)+half loop. Phase 2 in both learn and infer methods\n    # already predicts for timestep (t+1). We use that prediction for free and\n    # save the half-a-loop of work.\n\n    step = 0\n    while True:\n      # We get the prediction for the columns in the next time step from\n      # the topDownCompute method. It internally uses confidences.\n      multiStepColumnPredictions[step, :] = self.topDownCompute()\n\n      # Cleanest way in python to handle one and half loops\n      if step == nSteps-1:\n        break\n      step += 1\n\n      # Copy t-1 into t\n      self.infActiveState['t-1'][:, :] = self.infActiveState['t'][:, :]\n      self.infPredictedState['t-1'][:, :] = self.infPredictedState['t'][:, :]\n      self.cellConfidence['t-1'][:, :] = self.cellConfidence['t'][:, :]\n\n      # Predicted state at \"t-1\" becomes the active state at \"t\"\n      self.infActiveState['t'][:, :] = self.infPredictedState['t-1'][:, :]\n\n      # Predicted state and confidence are set in phase2.\n      self.infPredictedState['t'].fill(0)\n      self.cellConfidence['t'].fill(0.0)\n      self._inferPhase2()\n\n    # Revert the dynamic state to the saved state\n    self._setTPDynamicState(pristineTPDynamicState)\n\n    return multiStepColumnPredictions", "response": "This function computes the prediction for each column in the next time step."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a dict with all the dynamic state variables as keys and all the dynamic state values at this instant as values.", "response": "def _getTPDynamicState(self,):\n    \"\"\"\n    Parameters:\n    --------------------------------------------\n    retval:       A dict with all the dynamic state variable names as keys and\n                  their values at this instant as values.\n    \"\"\"\n    tpDynamicState = dict()\n    for variableName in self._getTPDynamicStateVariableNames():\n      tpDynamicState[variableName] = copy.deepcopy(self.__dict__[variableName])\n    return tpDynamicState"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nset all the dynamic state variables in the tm object with the values from the given dict.", "response": "def _setTPDynamicState(self, tpDynamicState):\n    \"\"\"\n    Set all the dynamic state variables from the <tpDynamicState> dict.\n\n    <tpDynamicState> dict has all the dynamic state variable names as keys and\n    their values at this instant as values.\n\n    We set the dynamic state variables in the tm object with these items.\n    \"\"\"\n    for variableName in self._getTPDynamicStateVariableNames():\n      self.__dict__[variableName] = tpDynamicState.pop(variableName)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _updateAvgLearnedSeqLength(self, prevSeqLength):\n    if self.lrnIterationIdx < 100:\n      alpha = 0.5\n    else:\n      alpha = 0.1\n\n    self.avgLearnedSeqLength = ((1.0 - alpha) * self.avgLearnedSeqLength +\n                                (alpha * prevSeqLength))", "response": "Update our moving average of learned sequence length."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ninfer backtrack of the current set of inputs.", "response": "def _inferBacktrack(self, activeColumns):\n    \"\"\"\n    This \"backtracks\" our inference state, trying to see if we can lock onto\n    the current set of inputs by assuming the sequence started up to N steps\n    ago on start cells.\n\n    This will adjust @ref infActiveState['t'] if it does manage to lock on to a\n    sequence that started earlier. It will also compute infPredictedState['t']\n    based on the possibly updated @ref infActiveState['t'], so there is no need to\n    call inferPhase2() after calling inferBacktrack().\n\n    This looks at:\n        - ``infActiveState['t']``\n\n    This updates/modifies:\n        - ``infActiveState['t']``\n        - ``infPredictedState['t']``\n        - ``colConfidence['t']``\n        - ``cellConfidence['t']``\n\n    How it works:\n\n    This method gets called from :meth:`updateInferenceState` when we detect \n    either of the following two conditions:\n\n    #. The current bottom-up input had too many un-expected columns\n    #. We fail to generate a sufficient number of predicted columns for the\n       next time step.\n\n    Either of these two conditions indicate that we have fallen out of a\n    learned sequence.\n\n    Rather than simply \"giving up\" and bursting on the unexpected input\n    columns, a better approach is to see if perhaps we are in a sequence that\n    started a few steps ago. The real world analogy is that you are driving\n    along and suddenly hit a dead-end, you will typically go back a few turns\n    ago and pick up again from a familiar intersection.\n\n    This back-tracking goes hand in hand with our learning methodology, which\n    always tries to learn again from start cells after it loses context. This\n    results in a network that has learned multiple, overlapping paths through\n    the input data, each starting at different points. The lower the global\n    decay and the more repeatability in the data, the longer each of these\n    paths will end up being.\n\n    The goal of this function is to find out which starting point in the past\n    leads to the current input with the most context as possible. This gives us\n    the best chance of predicting accurately going forward. Consider the\n    following example, where you have learned the following sub-sequences which\n    have the given frequencies:\n\n    ::\n\n                  ? - Q - C - D - E      10X      seq 0\n                  ? - B - C - D - F      1X       seq 1\n                  ? - B - C - H - I      2X       seq 2\n                  ? - B - C - D - F      3X       seq 3\n          ? - Z - A - B - C - D - J      2X       seq 4\n          ? - Z - A - B - C - H - I      1X       seq 5\n          ? - Y - A - B - C - D - F      3X       seq 6\n  \n        ----------------------------------------\n      W - X - Z - A - B - C - D          <= input history\n                              ^\n                              current time step\n\n    Suppose, in the current time step, the input pattern is D and you have not\n    predicted D, so you need to backtrack. Suppose we can backtrack up to 6\n    steps in the past, which path should we choose? From the table above, we can\n    see that the correct answer is to assume we are in seq 4. How do we\n    implement the backtrack to give us this right answer? The current\n    implementation takes the following approach:\n\n    #. Start from the farthest point in the past.\n    #. For each starting point S, calculate the confidence of the current\n       input, conf(startingPoint=S), assuming we followed that sequence.\n       Note that we must have learned at least one sequence that starts at\n       point S.\n    #. If conf(startingPoint=S) is significantly different from\n       conf(startingPoint=S-1), then choose S-1 as the starting point.\n\n    The assumption here is that starting point S-1 is the starting point of\n    a learned sub-sequence that includes the current input in it's path and\n    that started the longest ago. It thus has the most context and will be\n    the best predictor going forward.\n\n    From the statistics in the above table, we can compute what the confidences\n    will be for each possible starting point:\n\n    ::\n  \n      startingPoint           confidence of D\n      -----------------------------------------\n      B (t-2)               4/6  = 0.667   (seq 1,3)/(seq 1,2,3)\n      Z (t-4)               2/3  = 0.667   (seq 4)/(seq 4,5)\n\n    First of all, we do not compute any confidences at starting points t-1, t-3,\n    t-5, t-6 because there are no learned sequences that start at those points.\n\n    Notice here that Z is the starting point of the longest sub-sequence leading\n    up to the current input. Event though starting at t-2 and starting at t-4\n    give the same confidence value, we choose the sequence starting at t-4\n    because it gives the most context, and it mirrors the way that learning\n    extends sequences.\n\n    :param activeColumns: (list) of active column indices\n\n    \"\"\"\n    # How much input history have we accumulated?\n    # The current input is always at the end of self._prevInfPatterns (at\n    # index -1), but it is also evaluated as a potential starting point by\n    # turning on it's start cells and seeing if it generates sufficient\n    # predictions going forward.\n    numPrevPatterns = len(self._prevInfPatterns)\n    if numPrevPatterns <= 0:\n      return\n\n    # This is an easy to use label for the current time step\n    currentTimeStepsOffset = numPrevPatterns - 1\n\n    # Save our current active state in case we fail to find a place to restart\n    # todo: save infActiveState['t-1'], infPredictedState['t-1']?\n    self.infActiveState['backup'][:, :] = self.infActiveState['t'][:, :]\n\n    # Save our t-1 predicted state because we will write over it as as evaluate\n    # each potential starting point.\n    self.infPredictedState['backup'][:, :] = self.infPredictedState['t-1'][:, :]\n\n    # We will record which previous input patterns did not generate predictions\n    # up to the current time step and remove all the ones at the head of the\n    # input history queue so that we don't waste time evaluating them again at\n    # a later time step.\n    badPatterns = []\n\n    # Let's go back in time and replay the recent inputs from start cells and\n    #  see if we can lock onto this current set of inputs that way.\n    #\n    # Start the farthest back and work our way forward. For each starting point,\n    #  See if firing on start cells at that point would predict the current\n    #  input as well as generate sufficient predictions for the next time step.\n    #\n    # We want to pick the point closest to the current time step that gives us\n    # the relevant confidence. Think of this example, where we are at D and need\n    # to\n    #   A - B - C - D\n    # decide if we should backtrack to C, B, or A. Suppose B-C-D is a high order\n    # sequence and A is unrelated to it. If we backtrock to B would we get a\n    # certain confidence of D, but if went went farther back, to A, the\n    # confidence wouldn't change, since A has no impact on the B-C-D series.\n    #\n    # So, our strategy will be to pick the \"B\" point, since choosing the A point\n    #  does not impact our confidences going forward at all.\n    inSequence = False\n    candConfidence = None\n    candStartOffset = None\n    for startOffset in range(0, numPrevPatterns):\n\n      # If we have a candidate already in the past, don't bother falling back\n      #  to start cells on the current input.\n      if startOffset == currentTimeStepsOffset and candConfidence is not None:\n        break\n\n      if self.verbosity >= 3:\n        print (\n            \"Trying to lock-on using startCell state from %d steps ago:\" % (\n                numPrevPatterns - 1 - startOffset),\n            self._prevInfPatterns[startOffset])\n\n      # Play through starting from starting point 'startOffset'\n      inSequence = False\n      for offset in range(startOffset, numPrevPatterns):\n        # If we are about to set the active columns for the current time step\n        # based on what we predicted, capture and save the total confidence of\n        # predicting the current input\n        if offset == currentTimeStepsOffset:\n          totalConfidence = self.colConfidence['t'][activeColumns].sum()\n\n        # Compute activeState[t] given bottom-up and predictedState[t-1]\n        self.infPredictedState['t-1'][:, :] = self.infPredictedState['t'][:, :]\n        inSequence = self._inferPhase1(self._prevInfPatterns[offset],\n                                       useStartCells = (offset == startOffset))\n        if not inSequence:\n          break\n\n        # Compute predictedState['t'] given activeState['t']\n        if self.verbosity >= 3:\n          print (\"  backtrack: computing predictions from \",\n                 self._prevInfPatterns[offset])\n        inSequence = self._inferPhase2()\n        if not inSequence:\n          break\n\n      # If starting from startOffset got lost along the way, mark it as an\n      # invalid start point.\n      if not inSequence:\n        badPatterns.append(startOffset)\n        continue\n\n      # If we got to here, startOffset is a candidate starting point.\n      # Save this state as a candidate state. It will become the chosen state if\n      # we detect a change in confidences starting at a later startOffset\n      candConfidence = totalConfidence\n      candStartOffset = startOffset\n\n      if self.verbosity >= 3 and startOffset != currentTimeStepsOffset:\n        print (\"  # Prediction confidence of current input after starting %d \"\n               \"steps ago:\" % (numPrevPatterns - 1 - startOffset),\n               totalConfidence)\n\n      if candStartOffset == currentTimeStepsOffset:  # no more to try\n        break\n      self.infActiveState['candidate'][:, :] = self.infActiveState['t'][:, :]\n      self.infPredictedState['candidate'][:, :] = (\n          self.infPredictedState['t'][:, :])\n      self.cellConfidence['candidate'][:, :] = self.cellConfidence['t'][:, :]\n      self.colConfidence['candidate'][:] = self.colConfidence['t'][:]\n      break\n\n    # If we failed to lock on at any starting point, fall back to the original\n    # active state that we had on entry\n    if candStartOffset is None:\n      if self.verbosity >= 3:\n        print \"Failed to lock on. Falling back to bursting all unpredicted.\"\n      self.infActiveState['t'][:, :] = self.infActiveState['backup'][:, :]\n      self._inferPhase2()\n\n    else:\n      if self.verbosity >= 3:\n        print (\"Locked on to current input by using start cells from %d \"\n               \" steps ago:\" % (numPrevPatterns - 1 - candStartOffset),\n               self._prevInfPatterns[candStartOffset])\n      # Install the candidate state, if it wasn't the last one we evaluated.\n      if candStartOffset != currentTimeStepsOffset:\n        self.infActiveState['t'][:, :] = self.infActiveState['candidate'][:, :]\n        self.infPredictedState['t'][:, :] = (\n            self.infPredictedState['candidate'][:, :])\n        self.cellConfidence['t'][:, :] = self.cellConfidence['candidate'][:, :]\n        self.colConfidence['t'][:] = self.colConfidence['candidate'][:]\n\n    # Remove any useless patterns at the head of the previous input pattern\n    # queue.\n    for i in range(numPrevPatterns):\n      if (i in badPatterns or\n          (candStartOffset is not None and i <= candStartOffset)):\n        if self.verbosity >= 3:\n          print (\"Removing useless pattern from history:\",\n                 self._prevInfPatterns[0])\n        self._prevInfPatterns.pop(0)\n      else:\n        break\n\n    # Restore the original predicted state.\n    self.infPredictedState['t-1'][:, :] = self.infPredictedState['backup'][:, :]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _inferPhase1(self, activeColumns, useStartCells):\n    # Init to zeros to start\n    self.infActiveState['t'].fill(0)\n\n    # Phase 1 - turn on predicted cells in each column receiving bottom-up\n    # If we are following a reset, activate only the start cell in each\n    # column that has bottom-up\n    numPredictedColumns = 0\n    if useStartCells:\n      for c in activeColumns:\n        self.infActiveState['t'][c, 0] = 1\n\n    # else, turn on any predicted cells in each column. If there are none, then\n    # turn on all cells (burst the column)\n    else:\n      for c in activeColumns:\n        predictingCells = numpy.where(self.infPredictedState['t-1'][c] == 1)[0]\n        numPredictingCells = len(predictingCells)\n\n        if numPredictingCells > 0:\n          self.infActiveState['t'][c, predictingCells] = 1\n          numPredictedColumns += 1\n\n        else:\n          self.infActiveState['t'][c, :] = 1 # whole column bursts\n\n    # Did we predict this input well enough?\n    if useStartCells or numPredictedColumns >= 0.50 * len(activeColumns):\n      return True\n    else:\n      return False", "response": "Infer the current input from the last set of predictions and the current bottom - up."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _inferPhase2(self):\n    # Init to zeros to start\n    self.infPredictedState['t'].fill(0)\n    self.cellConfidence['t'].fill(0)\n    self.colConfidence['t'].fill(0)\n\n    # Phase 2 - Compute new predicted state and update cell and column\n    #   confidences\n    for c in xrange(self.numberOfCols):\n\n      # For each cell in the column\n      for i in xrange(self.cellsPerColumn):\n\n        # For each segment in the cell\n        for s in self.cells[c][i]:\n\n          # See if it has the min number of active synapses\n          numActiveSyns = self._getSegmentActivityLevel(\n              s, self.infActiveState['t'], connectedSynapsesOnly=False)\n          if numActiveSyns < self.activationThreshold:\n            continue\n\n          # Incorporate the confidence into the owner cell and column\n          if self.verbosity >= 6:\n            print \"incorporating DC from cell[%d,%d]:   \" % (c, i),\n            s.debugPrint()\n          dc = s.dutyCycle()\n          self.cellConfidence['t'][c, i] += dc\n          self.colConfidence['t'][c] += dc\n\n          # If we reach threshold on the connected synapses, predict it\n          # If not active, skip over it\n          if self._isSegmentActive(s, self.infActiveState['t']):\n            self.infPredictedState['t'][c, i] = 1\n\n    # Normalize column and cell confidences\n    sumConfidences = self.colConfidence['t'].sum()\n    if sumConfidences > 0:\n      self.colConfidence['t'] /= sumConfidences\n      self.cellConfidence['t'] /= sumConfidences\n\n    # Are we predicting the required minimum number of columns?\n    numPredictedCols = self.infPredictedState['t'].max(axis=1).sum()\n    if numPredictedCols >= 0.5 * self.avgInputDensity:\n      return True\n    else:\n      return False", "response": "Infer the next input from the current input columns."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _updateInferenceState(self, activeColumns):\n    # Copy t to t-1\n    self.infActiveState['t-1'][:, :] = self.infActiveState['t'][:, :]\n    self.infPredictedState['t-1'][:, :] = self.infPredictedState['t'][:, :]\n    self.cellConfidence['t-1'][:, :] = self.cellConfidence['t'][:, :]\n    self.colConfidence['t-1'][:] = self.colConfidence['t'][:]\n\n    # Each phase will zero/initilize the 't' states that it affects\n\n    # Update our inference input history\n    if self.maxInfBacktrack > 0:\n      if len(self._prevInfPatterns) > self.maxInfBacktrack:\n        self._prevInfPatterns.pop(0)\n      self._prevInfPatterns.append(activeColumns)\n\n    # Compute the active state given the predictions from last time step and\n    # the current bottom-up\n    inSequence = self._inferPhase1(activeColumns, self.resetCalled)\n\n    # If this input was considered unpredicted, let's go back in time and\n    # replay the recent inputs from start cells and see if we can lock onto\n    # this current set of inputs that way.\n    if not inSequence:\n      if self.verbosity >= 3:\n        print (\"Too much unpredicted input, re-tracing back to try and lock on \"\n               \"at an earlier timestep.\")\n      # inferBacktrack() will call inferPhase2() for us.\n      self._inferBacktrack(activeColumns)\n      return\n\n    # Compute the predicted cells and the cell and column confidences\n    inSequence = self._inferPhase2()\n    if not inSequence:\n      if self.verbosity >= 3:\n        print (\"Not enough predictions going forward, \"\n               \"re-tracing back to try and lock on at an earlier timestep.\")\n      # inferBacktrack() will call inferPhase2() for us.\n      self._inferBacktrack(activeColumns)", "response": "Update the inference state. Called from compute on every iteration."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _learnBacktrack(self):\n    # How much input history have we accumulated?\n    # The current input is always at the end of self._prevInfPatterns (at\n    # index -1), and is not a valid startingOffset to evaluate.\n    numPrevPatterns = len(self._prevLrnPatterns) - 1\n    if numPrevPatterns <= 0:\n      if self.verbosity >= 3:\n        print \"lrnBacktrack: No available history to backtrack from\"\n      return False\n\n    # We will record which previous input patterns did not generate predictions\n    # up to the current time step and remove all the ones at the head of the\n    # input history queue so that we don't waste time evaluating them again at\n    # a later time step.\n    badPatterns = []\n\n    # Let's go back in time and replay the recent inputs from start cells and\n    # see if we can lock onto this current set of inputs that way.\n    #\n    # Start the farthest back and work our way forward. For each starting point,\n    # See if firing on start cells at that point would predict the current\n    # input.\n    #\n    # We want to pick the point farthest in the past that has continuity\n    # up to the current time step\n    inSequence = False\n    for startOffset in range(0, numPrevPatterns):\n      # Can we backtrack from startOffset?\n      inSequence = self._learnBacktrackFrom(startOffset, readOnly=True)\n\n      # Done playing through the sequence from starting point startOffset\n      # Break out as soon as we find a good path\n      if inSequence:\n        break\n\n      # Take this bad starting point out of our input history so we don't\n      # try it again later.\n      badPatterns.append(startOffset)\n\n    # If we failed to lock on at any starting point, return failure. The caller\n    # will start over again on start cells\n    if not inSequence:\n      if self.verbosity >= 3:\n        print (\"Failed to lock on. Falling back to start cells on current \"\n               \"time step.\")\n      # Nothing in our input history was a valid starting point, so get rid\n      #  of it so we don't try any of them again at a later iteration\n      self._prevLrnPatterns = []\n      return False\n\n    # We did find a valid starting point in the past. Now, we need to\n    # re-enforce all segments that became active when following this path.\n    if self.verbosity >= 3:\n      print (\"Discovered path to current input by using start cells from %d \"\n             \"steps ago:\" % (numPrevPatterns - startOffset),\n             self._prevLrnPatterns[startOffset])\n\n    self._learnBacktrackFrom(startOffset, readOnly=False)\n\n    # Remove any useless patterns at the head of the input pattern history\n    # queue.\n    for i in range(numPrevPatterns):\n      if i in badPatterns or i <= startOffset:\n        if self.verbosity >= 3:\n          print (\"Removing useless pattern from history:\",\n                 self._prevLrnPatterns[0])\n        self._prevLrnPatterns.pop(0)\n      else:\n        break\n\n    return numPrevPatterns - startOffset", "response": "This function is called by _learnRandomly if we can backtrack the current set of inputs."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _learnPhase1(self, activeColumns, readOnly=False):\n    # Save previous active state and start out on a clean slate\n    self.lrnActiveState['t'].fill(0)\n\n    # For each column, turn on the predicted cell. There will always be at most\n    # one predicted cell per column\n    numUnpredictedColumns = 0\n    for c in activeColumns:\n      predictingCells = numpy.where(self.lrnPredictedState['t-1'][c] == 1)[0]\n      numPredictedCells = len(predictingCells)\n      assert numPredictedCells <= 1\n\n      # If we have a predicted cell, turn it on. The segment's posActivation\n      # count will have already been incremented by processSegmentUpdates\n      if numPredictedCells == 1:\n        i = predictingCells[0]\n        self.lrnActiveState['t'][c, i] = 1\n        continue\n\n      numUnpredictedColumns += 1\n      if readOnly:\n        continue\n\n      # If no predicted cell, pick the closest matching one to reinforce, or\n      # if none exists, create a new segment on a cell in that column\n      i, s, numActive = self._getBestMatchingCell(\n          c, self.lrnActiveState['t-1'], self.minThreshold)\n      if s is not None and s.isSequenceSegment():\n        if self.verbosity >= 4:\n          print \"Learn branch 0, found segment match. Learning on col=\", c\n        self.lrnActiveState['t'][c, i] = 1\n        segUpdate = self._getSegmentActiveSynapses(\n            c, i, s, self.lrnActiveState['t-1'], newSynapses = True)\n        s.totalActivations += 1\n        # This will update the permanences, posActivationsCount, and the\n        # lastActiveIteration (age).\n        trimSegment = self._adaptSegment(segUpdate)\n        if trimSegment:\n          self._trimSegmentsInCell(c, i, [s], minPermanence = 0.00001,\n                                   minNumSyns = 0)\n\n      # If no close match exists, create a new one\n      else:\n        # Choose a cell in this column to add a new segment to\n        i = self._getCellForNewSegment(c)\n        if (self.verbosity >= 4):\n          print \"Learn branch 1, no match. Learning on col=\", c,\n          print \", newCellIdxInCol=\", i\n        self.lrnActiveState['t'][c, i] = 1\n        segUpdate = self._getSegmentActiveSynapses(\n            c, i, None, self.lrnActiveState['t-1'], newSynapses=True)\n        segUpdate.sequenceSegment = True # Make it a sequence segment\n        self._adaptSegment(segUpdate)  # No need to check whether perm reached 0\n\n    # Determine if we are out of sequence or not and reset our PAM counter\n    # if we are in sequence\n    numBottomUpColumns = len(activeColumns)\n    if numUnpredictedColumns < numBottomUpColumns / 2:\n      return True   # in sequence\n    else:\n      return False", "response": "Compute the learning active state given the predicted state and the bottom - up input."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncomputes the predicted segments given the current set of active cells. :param readOnly True if being called from backtracking logic. This tells us not to increment any segment duty cycles or queue up any updates. This computes the lrnPredictedState['t'] and queues up any segments that became active (and the list of active synapses for each segment) into the segmentUpdates queue This looks at: - @ref lrnActiveState['t'] This modifies: - @ref lrnPredictedState['t'] - @ref segmentUpdates", "response": "def _learnPhase2(self, readOnly=False):\n    \"\"\"\n    Compute the predicted segments given the current set of active cells.\n\n    :param readOnly       True if being called from backtracking logic.\n                          This tells us not to increment any segment\n                          duty cycles or queue up any updates.\n\n    This computes the lrnPredictedState['t'] and queues up any segments that\n    became active (and the list of active synapses for each segment) into\n    the segmentUpdates queue\n\n    This looks at:\n        - @ref lrnActiveState['t']\n\n    This modifies:\n        - @ref lrnPredictedState['t']\n        - @ref segmentUpdates\n    \"\"\"\n    # Clear out predicted state to start with\n    self.lrnPredictedState['t'].fill(0)\n\n    # Compute new predicted state. When computing predictions for\n    # phase 2, we predict at  most one cell per column (the one with the best\n    # matching segment).\n    for c in xrange(self.numberOfCols):\n\n      # Is there a cell predicted to turn on in this column?\n      i, s, numActive = self._getBestMatchingCell(\n          c, self.lrnActiveState['t'], minThreshold = self.activationThreshold)\n      if i is None:\n        continue\n\n      # Turn on the predicted state for the best matching cell and queue\n      #  the pertinent segment up for an update, which will get processed if\n      #  the cell receives bottom up in the future.\n      self.lrnPredictedState['t'][c, i] = 1\n      if readOnly:\n        continue\n\n      # Queue up this segment for updating\n      segUpdate = self._getSegmentActiveSynapses(\n          c, i, s, activeState=self.lrnActiveState['t'],\n          newSynapses=(numActive < self.newSynapseCount))\n\n      s.totalActivations += 1    # increment totalActivations\n      self._addToSegmentUpdates(c, i, segUpdate)\n\n      if self.doPooling:\n        # creates a new pooling segment if no best matching segment found\n        # sum(all synapses) >= minThreshold, \"weak\" activation\n        predSegment = self._getBestMatchingSegment(c, i,\n                                                   self.lrnActiveState['t-1'])\n        segUpdate = self._getSegmentActiveSynapses(c, i, predSegment,\n                                                   self.lrnActiveState['t-1'], newSynapses=True)\n        self._addToSegmentUpdates(c, i, segUpdate)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _updateLearningState(self, activeColumns):\n    # Copy predicted and active states into t-1\n    self.lrnPredictedState['t-1'][:, :] = self.lrnPredictedState['t'][:, :]\n    self.lrnActiveState['t-1'][:, :] = self.lrnActiveState['t'][:, :]\n\n    # Update our learning input history\n    if self.maxLrnBacktrack > 0:\n      if len(self._prevLrnPatterns) > self.maxLrnBacktrack:\n        self._prevLrnPatterns.pop(0)\n      self._prevLrnPatterns.append(activeColumns)\n      if self.verbosity >= 4:\n        print \"Previous learn patterns: \\n\"\n        print self._prevLrnPatterns\n\n    # Process queued up segment updates, now that we have bottom-up, we\n    # can update the permanences on the cells that we predicted to turn on\n    # and did receive bottom-up\n    self._processSegmentUpdates(activeColumns)\n\n    # Decrement the PAM counter if it is running and increment our learned\n    # sequence length\n    if self.pamCounter > 0:\n      self.pamCounter -= 1\n    self.learnedSeqLength += 1\n\n    # Phase 1 - turn on the predicted cell in each column that received\n    # bottom-up. If there was no predicted cell, pick one to learn to.\n    if not self.resetCalled:\n      # Uses lrnActiveState['t-1'] and lrnPredictedState['t-1']\n      # computes lrnActiveState['t']\n      inSequence = self._learnPhase1(activeColumns)\n\n      # Reset our PAM counter if we are in sequence\n      if inSequence:\n        self.pamCounter = self.pamLength\n\n    # Print status of PAM counter, learned sequence length\n    if self.verbosity >= 3:\n      print \"pamCounter = \", self.pamCounter, \"seqLength = \", \\\n          self.learnedSeqLength\n\n    # Start over on start cells if any of the following occur:\n    #  1.) A reset was just called\n    #  2.) We have been loo long out of sequence (the pamCounter has expired)\n    #  3.) We have reached maximum allowed sequence length.\n    #\n    # Note that, unless we are following a reset, we also just learned or\n    # re-enforced connections to the current set of active columns because\n    # this input is still a valid prediction to learn.\n    #\n    # It is especially helpful to learn the connections to this input when\n    # you have a maxSeqLength constraint in place. Otherwise, you will have\n    # no continuity at all between sub-sequences of length maxSeqLength.\n    if (self.resetCalled or self.pamCounter == 0 or\n        (self.maxSeqLength != 0 and\n         self.learnedSeqLength >= self.maxSeqLength)):\n      if  self.verbosity >= 3:\n        if self.resetCalled:\n          print \"Starting over:\", activeColumns, \"(reset was called)\"\n        elif self.pamCounter == 0:\n          print \"Starting over:\", activeColumns, \"(PAM counter expired)\"\n        else:\n          print \"Starting over:\", activeColumns, \"(reached maxSeqLength)\"\n\n      # Update average learned sequence length - this is a diagnostic statistic\n      if self.pamCounter == 0:\n        seqLength = self.learnedSeqLength - self.pamLength\n      else:\n        seqLength = self.learnedSeqLength\n      if  self.verbosity >= 3:\n        print \"  learned sequence length was:\", seqLength\n      self._updateAvgLearnedSeqLength(seqLength)\n\n      # Backtrack to an earlier starting point, if we find one\n      backSteps = 0\n      if not self.resetCalled:\n        backSteps = self._learnBacktrack()\n\n      # Start over in the current time step if reset was called, or we couldn't\n      # backtrack.\n      if self.resetCalled or backSteps is None or backSteps == 0:\n        backSteps = 0\n        self.lrnActiveState['t'].fill(0)\n        for c in activeColumns:\n          self.lrnActiveState['t'][c, 0] = 1\n\n        # Remove any old input history patterns\n        self._prevLrnPatterns = []\n\n      # Reset PAM counter\n      self.pamCounter =  self.pamLength\n      self.learnedSeqLength = backSteps\n\n      # Clear out any old segment updates from prior sequences\n      self.segmentUpdates = {}\n\n    # Phase 2 - Compute new predicted state. When computing predictions for\n    # phase 2, we predict at  most one cell per column (the one with the best\n    # matching segment).\n    self._learnPhase2()", "response": "Update the learning state. Called from compute on every iteration."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncomputing the segment duty cycle of the given input.", "response": "def compute(self, bottomUpInput, enableLearn, enableInference=None):\n    \"\"\"\n    Handle one compute, possibly learning.\n\n    .. note::  It is an error to have both ``enableLearn`` and \n               ``enableInference`` set to False\n\n    .. note:: By default, we don't compute the inference output when learning \n              because it slows things down, but you can override this by passing \n              in True for ``enableInference``.\n\n    :param bottomUpInput: The bottom-up input as numpy list, typically from a \n           spatial pooler.\n    :param enableLearn: (bool) If true, perform learning\n    :param enableInference: (bool) If None, default behavior is to disable the \n           inference output when ``enableLearn`` is on. If true, compute the \n           inference output. If false, do not compute the inference output.\n\n    :returns: TODO: document\n\n    \"\"\"\n    # As a speed optimization for now (until we need online learning), skip\n    # computing the inference output while learning\n    if enableInference is None:\n      if enableLearn:\n        enableInference = False\n      else:\n        enableInference = True\n\n    assert (enableLearn or enableInference)\n\n    # Get the list of columns that have bottom-up\n    activeColumns = bottomUpInput.nonzero()[0]\n    if enableLearn:\n      self.lrnIterationIdx += 1\n    self.iterationIdx +=  1\n\n    if self.verbosity >= 3:\n      print \"\\n==== PY Iteration: %d =====\" % (self.iterationIdx)\n      print \"Active cols:\", activeColumns\n\n    # Update segment duty cycles if we are crossing a \"tier\"\n    # We determine if it's time to update the segment duty cycles. Since the\n    # duty cycle calculation is a moving average based on a tiered alpha, it is\n    # important that we update all segments on each tier boundary\n    if enableLearn:\n      if self.lrnIterationIdx in Segment.dutyCycleTiers:\n        for c, i in itertools.product(xrange(self.numberOfCols),\n                                      xrange(self.cellsPerColumn)):\n          for segment in self.cells[c][i]:\n            segment.dutyCycle()\n\n    # Update the average input density\n    if self.avgInputDensity is None:\n      self.avgInputDensity = len(activeColumns)\n    else:\n      self.avgInputDensity = (0.99 * self.avgInputDensity +\n                              0.01 * len(activeColumns))\n\n    # First, update the inference state\n    # As a speed optimization for now (until we need online learning), skip\n    # computing the inference output while learning\n    if enableInference:\n      self._updateInferenceState(activeColumns)\n\n    # Next, update the learning state\n    if enableLearn:\n      self._updateLearningState(activeColumns)\n\n      # Apply global decay, and remove synapses and/or segments.\n      # Synapses are removed if their permanence value is <= 0.\n      # Segments are removed when they don't have synapses anymore.\n      # Removal of synapses can trigger removal of whole segments!\n      # todo: isolate the synapse/segment retraction logic so that\n      # it can be called in adaptSegments, in the case where we\n      # do global decay only episodically.\n      if self.globalDecay > 0.0 and ((self.lrnIterationIdx % self.maxAge) == 0):\n        for c, i in itertools.product(xrange(self.numberOfCols),\n                                      xrange(self.cellsPerColumn)):\n\n          segsToDel = [] # collect and remove outside the loop\n          for segment in self.cells[c][i]:\n            age = self.lrnIterationIdx - segment.lastActiveIteration\n            if age <= self.maxAge:\n              continue\n\n            synsToDel = [] # collect and remove outside the loop\n            for synapse in segment.syns:\n\n              synapse[2] = synapse[2] - self.globalDecay # decrease permanence\n\n              if synapse[2] <= 0:\n                synsToDel.append(synapse) # add to list to delete\n\n            # 1 for sequenceSegment flag\n            if len(synsToDel) == segment.getNumSynapses():\n              segsToDel.append(segment) # will remove the whole segment\n            elif len(synsToDel) > 0:\n              for syn in synsToDel: # remove some synapses on segment\n                segment.syns.remove(syn)\n\n          for seg in segsToDel: # remove some segments of this cell\n            self._cleanUpdatesList(c, i, seg)\n            self.cells[c][i].remove(seg)\n\n    # Update the prediction score stats\n    # Learning always includes inference\n    if self.collectStats:\n      if enableInference:\n        predictedState = self.infPredictedState['t-1']\n      else:\n        predictedState = self.lrnPredictedState['t-1']\n      self._updateStatsInferEnd(self._internalStats,\n                                activeColumns,\n                                predictedState,\n                                self.colConfidence['t-1'])\n\n    # Finally return the TM output\n    output = self._computeOutput()\n\n    # Print diagnostic information based on the current verbosity level\n    self.printComputeEnd(output, learn=enableLearn)\n\n    self.resetCalled = False\n    return output"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef learn(self, bottomUpInput, enableInference=None):\n    return self.compute(bottomUpInput, enableLearn=True,\n                        enableInference=enableInference)", "response": "learn the sequence of tokens in the input document"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef trimSegments(self, minPermanence=None, minNumSyns=None):\n    # Fill in defaults\n    if minPermanence is None:\n      minPermanence = self.connectedPerm\n    if minNumSyns is None:\n      minNumSyns = self.activationThreshold\n\n    # Loop through all cells\n    totalSegsRemoved, totalSynsRemoved = 0, 0\n    for c, i in itertools.product(xrange(self.numberOfCols),\n                                  xrange(self.cellsPerColumn)):\n\n      (segsRemoved, synsRemoved) = self._trimSegmentsInCell(\n          colIdx=c, cellIdx=i, segList=self.cells[c][i],\n          minPermanence=minPermanence, minNumSyns=minNumSyns)\n      totalSegsRemoved += segsRemoved\n      totalSynsRemoved += synsRemoved\n\n    # Print all cells if verbosity says to\n    if self.verbosity >= 5:\n      print \"Cells, all segments:\"\n      self.printCells(predictedOnly=False)\n\n    return totalSegsRemoved, totalSynsRemoved", "response": "This method removes all synapses from the segment list and returns the number of synapses that are removed from the segment list."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _cleanUpdatesList(self, col, cellIdx, seg):\n    # TODO: check if the situation described in the docstring above actually\n    #       occurs.\n    for key, updateList in self.segmentUpdates.iteritems():\n      c, i = key[0], key[1]\n      if c == col and i == cellIdx:\n        for update in updateList:\n          if update[1].segment == seg:\n            self._removeSegmentUpdate(update)", "response": "Removes any update that would be for the given col cellIdx segIdx."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncall when learning has been completed.", "response": "def finishLearning(self):\n    \"\"\"\n    Called when learning has been completed. This method just calls\n    :meth:`trimSegments` and then clears out caches.\n    \"\"\"\n    # Keep weakly formed synapses around because they contain confidence scores\n    # for paths out of learned sequenced and produce a better prediction than\n    # chance.\n    self.trimSegments(minPermanence=0.0001)\n\n    # Update all cached duty cycles for better performance right after loading\n    # in the trained network.\n    for c, i in itertools.product(xrange(self.numberOfCols),\n                                  xrange(self.cellsPerColumn)):\n      for segment in self.cells[c][i]:\n        segment.dutyCycle()\n\n    # For error checking purposes, make sure no start cell has incoming\n    # connections\n    if self.cellsPerColumn > 1:\n      for c in xrange(self.numberOfCols):\n        assert self.getNumSegmentsInCell(c, 0) == 0"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfind weakly activated cell in column with at least minThreshold activeCells.", "response": "def _getBestMatchingCell(self, c, activeState, minThreshold):\n    \"\"\"\n    Find weakly activated cell in column with at least minThreshold active\n    synapses.\n\n    :param c            which column to look at\n    :param activeState  the active cells\n    :param minThreshold minimum number of synapses required\n\n    :returns: tuple (cellIdx, segment, numActiveSynapses)\n    \"\"\"\n    # Collect all cells in column c that have at least minThreshold in the most\n    # activated segment\n    bestActivityInCol = minThreshold\n    bestSegIdxInCol = -1\n    bestCellInCol = -1\n\n    for i in xrange(self.cellsPerColumn):\n\n      maxSegActivity = 0\n      maxSegIdx = 0\n\n      for j, s in enumerate(self.cells[c][i]):\n\n        activity = self._getSegmentActivityLevel(s, activeState)\n\n        if activity > maxSegActivity:\n          maxSegActivity = activity\n          maxSegIdx = j\n\n      if maxSegActivity >= bestActivityInCol:\n        bestActivityInCol = maxSegActivity\n        bestSegIdxInCol = maxSegIdx\n        bestCellInCol = i\n\n    if bestCellInCol == -1:\n      return (None, None, None)\n    else:\n      return (bestCellInCol, self.cells[c][bestCellInCol][bestSegIdxInCol],\n                bestActivityInCol)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the index of a cell in this column which is a good candidate for adding a new segment.", "response": "def _getCellForNewSegment(self, colIdx):\n    \"\"\"\n    Return the index of a cell in this column which is a good candidate\n    for adding a new segment.\n\n    When we have fixed size resources in effect, we insure that we pick a\n    cell which does not already have the max number of allowed segments. If\n    none exists, we choose the least used segment in the column to re-allocate.\n\n    :param colIdx which column to look at\n    :returns: cell index\n    \"\"\"\n    # Not fixed size CLA, just choose a cell randomly\n    if self.maxSegmentsPerCell < 0:\n      if self.cellsPerColumn > 1:\n        # Don't ever choose the start cell (cell # 0) in each column\n        i = self._random.getUInt32(self.cellsPerColumn-1) + 1\n      else:\n        i = 0\n      return i\n\n    # Fixed size CLA, choose from among the cells that are below the maximum\n    # number of segments.\n    # NOTE: It is important NOT to always pick the cell with the fewest number\n    # of segments. The reason is that if we always do that, we are more likely\n    # to run into situations where we choose the same set of cell indices to\n    # represent an 'A' in both context 1 and context 2. This is because the\n    # cell indices we choose in each column of a pattern will advance in\n    # lockstep (i.e. we pick cell indices of 1, then cell indices of 2, etc.).\n    candidateCellIdxs = []\n    if self.cellsPerColumn == 1:\n      minIdx = 0\n      maxIdx = 0\n    else:\n      minIdx = 1                      # Don't include startCell in the mix\n      maxIdx = self.cellsPerColumn-1\n    for i in xrange(minIdx, maxIdx+1):\n      numSegs = len(self.cells[colIdx][i])\n      if numSegs < self.maxSegmentsPerCell:\n        candidateCellIdxs.append(i)\n\n    # If we found one, return with it. Note we need to use _random to maintain\n    # correspondence with CPP code.\n    if len(candidateCellIdxs) > 0:\n      #candidateCellIdx = random.choice(candidateCellIdxs)\n      candidateCellIdx = (\n          candidateCellIdxs[self._random.getUInt32(len(candidateCellIdxs))])\n      if self.verbosity >= 5:\n        print \"Cell [%d,%d] chosen for new segment, # of segs is %d\" % (\n            colIdx, candidateCellIdx, len(self.cells[colIdx][candidateCellIdx]))\n      return candidateCellIdx\n\n    # All cells in the column are full, find a segment to free up\n    candidateSegment = None\n    candidateSegmentDC = 1.0\n    # For each cell in this column\n    for i in xrange(minIdx, maxIdx+1):\n      # For each segment in this cell\n      for s in self.cells[colIdx][i]:\n        dc = s.dutyCycle()\n        if dc < candidateSegmentDC:\n          candidateCellIdx = i\n          candidateSegmentDC = dc\n          candidateSegment = s\n\n    # Free up the least used segment\n    if self.verbosity >= 5:\n      print (\"Deleting segment #%d for cell[%d,%d] to make room for new \"\n             \"segment\" % (candidateSegment.segID, colIdx, candidateCellIdx))\n      candidateSegment.debugPrint()\n    self._cleanUpdatesList(colIdx, candidateCellIdx, candidateSegment)\n    self.cells[colIdx][candidateCellIdx].remove(candidateSegment)\n    return candidateCellIdx"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a segmentUpdate data structure containing a list of proposed synapses changes to segment s.", "response": "def _getSegmentActiveSynapses(self, c, i, s, activeState, newSynapses=False):\n    \"\"\"\n    Return a segmentUpdate data structure containing a list of proposed\n    changes to segment s. Let activeSynapses be the list of active synapses\n    where the originating cells have their activeState output = 1 at time step\n    t. (This list is empty if s is None since the segment doesn't exist.)\n    newSynapses is an optional argument that defaults to false. If newSynapses\n    is true, then newSynapseCount - len(activeSynapses) synapses are added to\n    activeSynapses. These synapses are randomly chosen from the set of cells\n    that have learnState = 1 at timeStep.\n\n    :param c TODO: document\n    :param i TODO: document\n    :param s TODO: document\n    :param activeState TODO: document\n    :param newSynapses TODO: document\n    \"\"\"\n    activeSynapses = []\n\n    if s is not None: # s can be None, if adding a new segment\n      # Here we add *integers* to activeSynapses\n      activeSynapses = [idx for idx, syn in enumerate(s.syns) \\\n                        if activeState[syn[0], syn[1]]]\n\n    if newSynapses: # add a few more synapses\n\n      nSynapsesToAdd = self.newSynapseCount - len(activeSynapses)\n\n      # Here we add *pairs* (colIdx, cellIdx) to activeSynapses\n      activeSynapses += self._chooseCellsToLearnFrom(c, i, s, nSynapsesToAdd,\n                                                     activeState)\n\n    # It's still possible that activeSynapses is empty, and this will\n    # be handled in addToSegmentUpdates\n\n    # NOTE: activeSynapses contains a mixture of integers and pairs of integers\n    # - integers are indices of synapses already existing on the segment,\n    #   that we will need to update.\n    # - pairs represent source (colIdx, cellIdx) of new synapses to create on\n    #   the segment\n    update = BacktrackingTM._SegmentUpdate(c, i, s, activeSynapses)\n\n    return update"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchoosing n random cells to learn from. This function is called several times while learning with timeStep = t-1, so we cache the set of candidates for that case. It's also called once with timeStep = t, and we cache that set of candidates. :returns: tuple (column index, cell index).", "response": "def _chooseCellsToLearnFrom(self, c, i, s, n, activeState):\n    \"\"\"\n    Choose n random cells to learn from.\n\n    This function is called several times while learning with timeStep = t-1, so\n    we cache the set of candidates for that case. It's also called once with\n    timeStep = t, and we cache that set of candidates.\n\n    :returns: tuple (column index, cell index).\n    \"\"\"\n    if n <= 0:\n      return []\n\n    tmpCandidates = numpy.where(activeState == 1)\n\n    # Candidates can be empty at this point, in which case we return\n    # an empty segment list. adaptSegments will do nothing when getting\n    # that list.\n    if len(tmpCandidates[0]) == 0:\n      return []\n\n    if s is None: # new segment\n      cands = [syn for syn in zip(tmpCandidates[0], tmpCandidates[1])]\n    else:\n      # We exclude any synapse that is already in this segment.\n      synapsesAlreadyInSegment = set((syn[0], syn[1]) for syn in s.syns)\n      cands = [syn for syn in zip(tmpCandidates[0], tmpCandidates[1])\n               if (syn[0], syn[1]) not in synapsesAlreadyInSegment]\n\n    # If we have no more candidates than requested, return all of them,\n    # no shuffle necessary.\n    if len(cands) <= n:\n      return cands\n\n    if n == 1: # so that we don't shuffle if only one is needed\n      idx = self._random.getUInt32(len(cands))\n      return [cands[idx]]  # col and cell idx in col\n\n    # If we need more than one candidate\n    indices = numpy.array([j for j in range(len(cands))], dtype='uint32')\n    tmp = numpy.zeros(min(n, len(indices)), dtype='uint32')\n    self._random.sample(indices, tmp)\n    return sorted([cands[j] for j in tmp])"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _processSegmentUpdates(self, activeColumns):\n    # The segmentUpdates dict has keys which are the column,cellIdx of the\n    # owner cell. The values are lists of segment updates for that cell\n    removeKeys = []\n    trimSegments = []\n    for key, updateList in self.segmentUpdates.iteritems():\n\n      # Get the column number and cell index of the owner cell\n      c, i = key[0], key[1]\n\n      # If the cell received bottom-up, update its segments\n      if c in activeColumns:\n        action = 'update'\n\n      # If not, either keep it around if it's still predicted, or remove it\n      else:\n        # If it is still predicted, and we are pooling, keep it around\n        if self.doPooling and self.lrnPredictedState['t'][c, i] == 1:\n          action = 'keep'\n        else:\n          action = 'remove'\n\n      # Process each segment for this cell. Each segment entry contains\n      # [creationDate, SegmentInfo]\n      updateListKeep = []\n      if action != 'remove':\n        for (createDate, segUpdate) in updateList:\n\n          if self.verbosity >= 4:\n            print \"_nLrnIterations =\", self.lrnIterationIdx,\n            print segUpdate\n\n          # If this segment has expired. Ignore this update (and hence remove it\n          # from list)\n          if self.lrnIterationIdx - createDate > self.segUpdateValidDuration:\n            continue\n\n          if action == 'update':\n            trimSegment = self._adaptSegment(segUpdate)\n            if trimSegment:\n              trimSegments.append((segUpdate.columnIdx, segUpdate.cellIdx,\n                                        segUpdate.segment))\n          else:\n            # Keep segments that haven't expired yet (the cell is still being\n            #   predicted)\n            updateListKeep.append((createDate, segUpdate))\n\n      self.segmentUpdates[key] = updateListKeep\n      if len(updateListKeep) == 0:\n        removeKeys.append(key)\n\n    # Clean out empty segment updates\n    for key in removeKeys:\n      self.segmentUpdates.pop(key)\n\n    # Trim segments that had synapses go to 0\n    for (c, i, segment) in trimSegments:\n      self._trimSegmentsInCell(c, i, [segment], minPermanence = 0.00001,\n                               minNumSyns = 0)", "response": "Process the segment updates for the current active column and remove them if they are too old."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncomputing and return the positive activations duty cycle of this segment.", "response": "def dutyCycle(self, active=False, readOnly=False):\n    \"\"\"Compute/update and return the positive activations duty cycle of\n    this segment. This is a measure of how often this segment is\n    providing good predictions.\n\n    :param active   True if segment just provided a good prediction\n\n    :param readOnly If True, compute the updated duty cycle, but don't change\n               the cached value. This is used by debugging print statements.\n\n    :returns: The duty cycle, a measure of how often this segment is\n    providing good predictions.\n\n    **NOTE:** This method relies on different schemes to compute the duty cycle\n    based on how much history we have. In order to support this tiered\n    approach **IT MUST BE CALLED ON EVERY SEGMENT AT EACH DUTY CYCLE TIER**\n    (@ref dutyCycleTiers).\n\n    When we don't have a lot of history yet (first tier), we simply return\n    number of positive activations / total number of iterations\n\n    After a certain number of iterations have accumulated, it converts into\n    a moving average calculation, which is updated only when requested\n    since it can be a bit expensive to compute on every iteration (it uses\n    the pow() function).\n\n    The duty cycle is computed as follows:\n\n        dc[t] = (1-alpha) * dc[t-1] + alpha * value[t]\n\n    If the value[t] has been 0 for a number of steps in a row, you can apply\n    all of the updates at once using:\n\n        dc[t] = (1-alpha)^(t-lastT) * dc[lastT]\n\n    We use the alphas and tiers as defined in @ref dutyCycleAlphas and\n    @ref dutyCycleTiers.\n    \"\"\"\n    # For tier #0, compute it from total number of positive activations seen\n    if self.tm.lrnIterationIdx <= self.dutyCycleTiers[1]:\n      dutyCycle = float(self.positiveActivations) \\\n                                    / self.tm.lrnIterationIdx\n      if not readOnly:\n        self._lastPosDutyCycleIteration = self.tm.lrnIterationIdx\n        self._lastPosDutyCycle = dutyCycle\n      return dutyCycle\n\n    # How old is our update?\n    age = self.tm.lrnIterationIdx - self._lastPosDutyCycleIteration\n\n    # If it's already up to date, we can returned our cached value.\n    if age == 0 and not active:\n      return self._lastPosDutyCycle\n\n    # Figure out which alpha we're using\n    for tierIdx in range(len(self.dutyCycleTiers)-1, 0, -1):\n      if self.tm.lrnIterationIdx > self.dutyCycleTiers[tierIdx]:\n        alpha = self.dutyCycleAlphas[tierIdx]\n        break\n\n    # Update duty cycle\n    dutyCycle = pow(1.0-alpha, age) * self._lastPosDutyCycle\n    if active:\n      dutyCycle += alpha\n\n    # Update cached values if not read-only\n    if not readOnly:\n      self._lastPosDutyCycleIteration = self.tm.lrnIterationIdx\n      self._lastPosDutyCycle = dutyCycle\n\n    return dutyCycle"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef addSynapse(self, srcCellCol, srcCellIdx, perm):\n    self.syns.append([int(srcCellCol), int(srcCellIdx), numpy.float32(perm)])", "response": "Add a synapse to the synonym list"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef logExceptions(logger=None):\n  logger = (logger if logger is not None else logging.getLogger(__name__))\n\n  def exceptionLoggingDecorator(func):\n    @functools.wraps(func)\n    def exceptionLoggingWrap(*args, **kwargs):\n      try:\n        return func(*args, **kwargs)\n      except:\n        logger.exception(\n          \"Unhandled exception %r from %r. Caller stack:\\n%s\",\n          sys.exc_info()[1], func, ''.join(traceback.format_stack()), )\n        raise\n\n    return exceptionLoggingWrap\n\n  return exceptionLoggingDecorator", "response": "Returns a closure that can be used as function or method decorator for logging exceptions."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a closure that can be used as function or method decorator for logging entry and exit of a function.", "response": "def logEntryExit(getLoggerCallback=logging.getLogger,\n                 entryExitLogLevel=logging.DEBUG, logArgs=False,\n                 logTraceback=False):\n  \"\"\" Returns a closure suitable for use as function/method decorator for\n  logging entry/exit of function/method.\n\n  getLoggerCallback:    user-supplied callback function that takes no args and\n                          returns the logger instance to use for logging.\n  entryExitLogLevel:    Log level for logging entry/exit of decorated function;\n                          e.g., logging.DEBUG; pass None to disable entry/exit\n                          logging.\n  logArgs:              If True, also log args\n  logTraceback:         If True, also log Traceback information\n\n  Usage Examples:\n    NOTE: logging must be initialized *before* any loggers are created, else\n      there will be no output; see nupic.support.initLogging()\n\n    @logEntryExit()\n    def myFunctionBar():\n        ...\n\n\n    @logEntryExit(logTraceback=True)\n    @logExceptions()\n    def myFunctionGamma():\n        ...\n        raise RuntimeError(\"something bad happened\")\n        ...\n  \"\"\"\n\n  def entryExitLoggingDecorator(func):\n\n    @functools.wraps(func)\n    def entryExitLoggingWrap(*args, **kwargs):\n\n      if entryExitLogLevel is None:\n        enabled = False\n      else:\n        logger = getLoggerCallback()\n        enabled = logger.isEnabledFor(entryExitLogLevel)\n\n      if not enabled:\n        return func(*args, **kwargs)\n\n      funcName = str(func)\n\n      if logArgs:\n        argsRepr = ', '.join(\n          [repr(a) for a in args] +\n          ['%s=%r' % (k,v,) for k,v in kwargs.iteritems()])\n      else:\n        argsRepr = ''\n\n      logger.log(\n        entryExitLogLevel, \"ENTERING: %s(%s)%s\", funcName, argsRepr,\n        '' if not logTraceback else '; ' + repr(traceback.format_stack()))\n\n      try:\n        return func(*args, **kwargs)\n      finally:\n        logger.log(\n          entryExitLogLevel, \"LEAVING: %s(%s)%s\", funcName, argsRepr,\n          '' if not logTraceback else '; ' + repr(traceback.format_stack()))\n\n\n    return entryExitLoggingWrap\n\n  return entryExitLoggingDecorator"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a closure suitable for use as function/method decorator for retrying a function being decorated. timeoutSec: How many seconds from time of initial call to stop retrying (floating point); 0 = no retries initialRetryDelaySec: Number of seconds to wait for first retry. Subsequent retries will occur at geometrically doubling intervals up to a maximum interval of maxRetryDelaySec (floating point) maxRetryDelaySec: Maximum amount of seconds to wait between retries (floating point) retryExceptions: A tuple (must be a tuple) of exception classes that, including their subclasses, should trigger retries; Default: any Exception-based exception will trigger retries retryFilter: Optional filter function used to further filter the exceptions in the retryExceptions tuple; called if the current exception meets the retryExceptions criteria: takes the current exception instance, args, and kwargs that were passed to the decorated function, and returns True to retry, False to allow the exception to be re-raised without retrying. Default: permits any exception that matches retryExceptions to be retried. logger: User-supplied logger instance to use for logging. None=defaults to logging.getLogger(__name__). Usage Example: NOTE: logging must be initialized *before* any loggers are created, else there will be no output; see nupic.support.initLogging() _retry = retry(timeoutSec=300, initialRetryDelaySec=0.2, maxRetryDelaySec=10, retryExceptions=[socket.error]) @_retry def myFunctionFoo(): ... raise RuntimeError(\"something bad happened\") ...", "response": "def retry(timeoutSec, initialRetryDelaySec, maxRetryDelaySec,\n          retryExceptions=(Exception,),\n          retryFilter=lambda e, args, kwargs: True,\n          logger=None, clientLabel=\"\"):\n  \"\"\" Returns a closure suitable for use as function/method decorator for\n  retrying a function being decorated.\n\n  timeoutSec:           How many seconds from time of initial call to stop\n                        retrying (floating point); 0 = no retries\n  initialRetryDelaySec: Number of seconds to wait for first retry.\n                        Subsequent retries will occur at geometrically\n                        doubling intervals up to a maximum interval of\n                        maxRetryDelaySec (floating point)\n  maxRetryDelaySec:     Maximum amount of seconds to wait between retries\n                        (floating point)\n  retryExceptions:      A tuple (must be a tuple) of exception classes that,\n                        including their subclasses, should trigger retries;\n                        Default: any Exception-based exception will trigger\n                        retries\n  retryFilter:          Optional filter function used to further filter the\n                        exceptions in the retryExceptions tuple; called if the\n                        current exception meets the retryExceptions criteria:\n                        takes the current exception instance, args, and kwargs\n                        that were passed to the decorated function, and returns\n                        True to retry, False to allow the exception to be\n                        re-raised without retrying. Default: permits any\n                        exception that matches retryExceptions to be retried.\n  logger:               User-supplied logger instance to use for logging.\n                        None=defaults to logging.getLogger(__name__).\n\n  Usage Example:\n    NOTE: logging must be initialized *before* any loggers are created, else\n      there will be no output; see nupic.support.initLogging()\n\n    _retry = retry(timeoutSec=300, initialRetryDelaySec=0.2,\n                   maxRetryDelaySec=10, retryExceptions=[socket.error])\n    @_retry\n    def myFunctionFoo():\n        ...\n        raise RuntimeError(\"something bad happened\")\n        ...\n  \"\"\"\n\n  assert initialRetryDelaySec > 0, str(initialRetryDelaySec)\n\n  assert timeoutSec >= 0, str(timeoutSec)\n\n  assert maxRetryDelaySec >= initialRetryDelaySec, \\\n      \"%r < %r\" % (maxRetryDelaySec, initialRetryDelaySec)\n\n  assert isinstance(retryExceptions, tuple), (\n    \"retryExceptions must be tuple, but got %r\") % (type(retryExceptions),)\n\n  if logger is None:\n    logger = logging.getLogger(__name__)\n\n  def retryDecorator(func):\n    @functools.wraps(func)\n    def retryWrap(*args, **kwargs):\n      numAttempts = 0\n      delaySec = initialRetryDelaySec\n      startTime = time.time()\n\n      # Make sure it gets called at least once\n      while True:\n        numAttempts += 1\n        try:\n          result = func(*args, **kwargs)\n        except retryExceptions, e:\n          if not retryFilter(e, args, kwargs):\n            if logger.isEnabledFor(logging.DEBUG):\n              logger.debug(\n                '[%s] Failure in %r; retries aborted by custom retryFilter. '\n                'Caller stack:\\n%s', clientLabel, func,\n                ''.join(traceback.format_stack()), exc_info=True)\n            raise\n\n          now = time.time()\n          # Compensate for negative time adjustment so we don't get stuck\n          # waiting way too long (python doesn't provide monotonic time yet)\n          if now < startTime:\n            startTime = now\n          if (now - startTime) >= timeoutSec:\n            logger.exception(\n              '[%s] Exhausted retry timeout (%s sec.; %s attempts) for %r. '\n              'Caller stack:\\n%s', clientLabel, timeoutSec, numAttempts, func,\n              ''.join(traceback.format_stack()))\n            raise\n\n          if numAttempts == 1:\n            logger.warning(\n              '[%s] First failure in %r; initial retry in %s sec.; '\n              'timeoutSec=%s. Caller stack:\\n%s', clientLabel, func, delaySec,\n              timeoutSec, ''.join(traceback.format_stack()), exc_info=True)\n          else:\n            logger.debug(\n              '[%s] %r failed %s times; retrying in %s sec.; timeoutSec=%s. '\n              'Caller stack:\\n%s',\n              clientLabel, func, numAttempts, delaySec, timeoutSec,\n              ''.join(traceback.format_stack()), exc_info=True)\n          time.sleep(delaySec)\n\n          delaySec = min(delaySec*2, maxRetryDelaySec)\n        else:\n          if numAttempts > 1:\n            logger.info('[%s] %r succeeded on attempt # %d',\n                        clientLabel, func, numAttempts)\n\n          return result\n\n    return retryWrap\n\n  return retryDecorator"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of simple patterns. Each pattern has numOnes consecutive patterns and numPatterns consecutive patterns have numOnes consecutive patterns.", "response": "def getSimplePatterns(numOnes, numPatterns, patternOverlap=0):\n  \"\"\"Very simple patterns. Each pattern has numOnes consecutive\n  bits on. The amount of overlap between consecutive patterns is\n  configurable, via the patternOverlap parameter.\n\n  Parameters:\n  -----------------------------------------------------------------------\n  numOnes:        Number of bits ON in each pattern\n  numPatterns:    Number of unique patterns to generate\n  patternOverlap: Number of bits of overlap between each successive pattern\n  retval:         patterns\n  \"\"\"\n\n  assert (patternOverlap < numOnes)\n\n  # How many new bits are introduced in each successive pattern?\n  numNewBitsInEachPattern = numOnes - patternOverlap\n  numCols = numNewBitsInEachPattern * numPatterns + patternOverlap\n\n  p = []\n  for i in xrange(numPatterns):\n    x = numpy.zeros(numCols, dtype='float32')\n\n    startBit = i*numNewBitsInEachPattern\n    nextStartBit = startBit + numOnes\n    x[startBit:nextStartBit] = 1\n\n    p.append(x)\n\n  return p"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nbuilds the training sequences that share some elements in the middle of the sequence.", "response": "def buildOverlappedSequences( numSequences = 2,\n                              seqLen = 5,\n                              sharedElements = [3,4],\n                              numOnBitsPerPattern = 3,\n                              patternOverlap = 0,\n                              seqOverlap = 0,\n                              **kwargs\n                              ):\n  \"\"\" Create training sequences that share some elements in the middle.\n\n  Parameters:\n  -----------------------------------------------------\n  numSequences:         Number of unique training sequences to generate\n  seqLen:               Overall length of each sequence\n  sharedElements:       Which element indices of each sequence are shared. These\n                          will be in the range between 0 and seqLen-1\n  numOnBitsPerPattern:  Number of ON bits in each TM input pattern\n  patternOverlap:       Max number of bits of overlap between any 2 patterns\n  retval:               (numCols, trainingSequences)\n                          numCols - width of the patterns\n                          trainingSequences - a list of training sequences\n\n  \"\"\"\n\n  # Total number of patterns used to build the sequences\n  numSharedElements = len(sharedElements)\n  numUniqueElements = seqLen - numSharedElements\n  numPatterns = numSharedElements + numUniqueElements * numSequences\n\n  # Create the table of patterns\n  patterns = getSimplePatterns(numOnBitsPerPattern, numPatterns, patternOverlap)\n\n  # Total number of columns required\n  numCols = len(patterns[0])\n\n\n  # -----------------------------------------------------------------------\n  # Create the training sequences\n  trainingSequences = []\n\n  uniquePatternIndices = range(numSharedElements, numPatterns)\n  for _ in xrange(numSequences):\n    sequence = []\n\n    # pattern indices [0 ... numSharedElements-1] are reserved for the shared\n    #  middle\n    sharedPatternIndices = range(numSharedElements)\n\n    # Build up the sequence\n    for j in xrange(seqLen):\n      if j in sharedElements:\n        patIdx = sharedPatternIndices.pop(0)\n      else:\n        patIdx = uniquePatternIndices.pop(0)\n      sequence.append(patterns[patIdx])\n\n    trainingSequences.append(sequence)\n\n\n  if VERBOSITY >= 3:\n    print \"\\nTraining sequences\"\n    printAllTrainingSequences(trainingSequences)\n\n  return (numCols, trainingSequences)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef buildSequencePool(numSequences = 10,\n                      seqLen = [2,3,4],\n                      numPatterns = 5,\n                      numOnBitsPerPattern = 3,\n                      patternOverlap = 0,\n                      **kwargs\n                      ):\n  \"\"\" Create a bunch of sequences of various lengths, all built from\n  a fixed set of patterns.\n\n  Parameters:\n  -----------------------------------------------------\n  numSequences:         Number of training sequences to generate\n  seqLen:               List of possible sequence lengths\n  numPatterns:          How many possible patterns there are to use within\n                          sequences\n  numOnBitsPerPattern:  Number of ON bits in each TM input pattern\n  patternOverlap:       Max number of bits of overlap between any 2 patterns\n  retval:               (numCols, trainingSequences)\n                          numCols - width of the patterns\n                          trainingSequences - a list of training sequences\n\n  \"\"\"\n\n\n  # Create the table of patterns\n  patterns = getSimplePatterns(numOnBitsPerPattern, numPatterns, patternOverlap)\n\n  # Total number of columns required\n  numCols = len(patterns[0])\n\n\n  # -----------------------------------------------------------------------\n  # Create the training sequences\n  trainingSequences = []\n  for _ in xrange(numSequences):\n\n    # Build it up from patterns\n    sequence = []\n    length = random.choice(seqLen)\n    for _ in xrange(length):\n      patIdx = random.choice(xrange(numPatterns))\n      sequence.append(patterns[patIdx])\n\n    # Put it in\n    trainingSequences.append(sequence)\n\n\n  if VERBOSITY >= 3:\n    print \"\\nTraining sequences\"\n    printAllTrainingSequences(trainingSequences)\n\n  return (numCols, trainingSequences)", "response": "Builds a bunch of training sequences from a fixed set of patterns."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef createTMs(includeCPP = True,\n              includePy = True,\n              numCols = 100,\n              cellsPerCol = 4,\n              activationThreshold = 3,\n              minThreshold = 3,\n              newSynapseCount = 3,\n              initialPerm = 0.6,\n              permanenceInc = 0.1,\n              permanenceDec = 0.0,\n              globalDecay = 0.0,\n              pamLength = 0,\n              checkSynapseConsistency = True,\n              maxInfBacktrack = 0,\n              maxLrnBacktrack = 0,\n              **kwargs\n              ):\n\n  \"\"\"Create one or more TM instances, placing each into a dict keyed by\n  name.\n\n  Parameters:\n  ------------------------------------------------------------------\n  retval:   tms - dict of TM instances\n  \"\"\"\n\n  # Keep these fixed:\n  connectedPerm = 0.5\n\n  tms = dict()\n\n  if includeCPP:\n    if VERBOSITY >= 2:\n      print \"Creating BacktrackingTMCPP instance\"\n\n    cpp_tm = BacktrackingTMCPP(numberOfCols = numCols, cellsPerColumn = cellsPerCol,\n                               initialPerm = initialPerm, connectedPerm = connectedPerm,\n                               minThreshold = minThreshold, newSynapseCount = newSynapseCount,\n                               permanenceInc = permanenceInc, permanenceDec = permanenceDec,\n                               activationThreshold = activationThreshold,\n                               globalDecay = globalDecay, burnIn = 1,\n                               seed=SEED, verbosity=VERBOSITY,\n                               checkSynapseConsistency = checkSynapseConsistency,\n                               collectStats = True,\n                               pamLength = pamLength,\n                               maxInfBacktrack = maxInfBacktrack,\n                               maxLrnBacktrack = maxLrnBacktrack,\n                               )\n\n    # Ensure we are copying over learning states for TMDiff\n    cpp_tm.retrieveLearningStates = True\n\n    tms['CPP'] = cpp_tm\n\n\n  if includePy:\n    if VERBOSITY >= 2:\n      print \"Creating PY TM instance\"\n\n    py_tm = BacktrackingTM(numberOfCols = numCols,\n                           cellsPerColumn = cellsPerCol,\n                           initialPerm = initialPerm,\n                           connectedPerm = connectedPerm,\n                           minThreshold = minThreshold,\n                           newSynapseCount = newSynapseCount,\n                           permanenceInc = permanenceInc,\n                           permanenceDec = permanenceDec,\n                           activationThreshold = activationThreshold,\n                           globalDecay = globalDecay, burnIn = 1,\n                           seed=SEED, verbosity=VERBOSITY,\n                           collectStats = True,\n                           pamLength = pamLength,\n                           maxInfBacktrack = maxInfBacktrack,\n                           maxLrnBacktrack = maxLrnBacktrack,\n                           )\n\n\n    tms['PY '] = py_tm\n\n  return tms", "response": "Creates one or more BacktrackingTM objects for each column."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchecks for diffs among the TM instances in the passed in TMs dict and raise an assert if any are detected", "response": "def assertNoTMDiffs(tms):\n  \"\"\"\n  Check for diffs among the TM instances in the passed in tms dict and\n  raise an assert if any are detected\n\n  Parameters:\n  ---------------------------------------------------------------------\n  tms:                  dict of TM instances\n  \"\"\"\n\n  if len(tms) == 1:\n    return\n  if len(tms) > 2:\n    raise \"Not implemented for more than 2 TMs\"\n\n  same = fdrutils.tmDiff2(tms.values(), verbosity=VERBOSITY)\n  assert(same)\n  return"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nevaluate the training set for nTrainRepetitions times and return inference stats.", "response": "def evalSequences(tms,\n                  trainingSequences,\n                  testSequences = None,\n                  nTrainRepetitions = 1,\n                  doResets = True,\n                  **kwargs):\n\n  \"\"\"Train the TMs on the entire training set for nTrainRepetitions in a row.\n  Then run the test set through inference once and return the inference stats.\n\n  Parameters:\n  ---------------------------------------------------------------------\n  tms:                  dict of TM instances\n  trainingSequences:    list of training sequences. Each sequence is a list\n                        of TM input patterns\n  testSequences:        list of test sequences. If None, we will test against\n                        the trainingSequences\n  nTrainRepetitions:    Number of times to run the training set through the TM\n  doResets:             If true, send a reset to the TM between each sequence\n  \"\"\"\n\n  # If no test sequence is specified, use the first training sequence\n  if testSequences == None:\n    testSequences = trainingSequences\n\n  # First TM instance is used by default for verbose printing of input values,\n  #  etc.\n  firstTM = tms.values()[0]\n\n  assertNoTMDiffs(tms)\n\n  # =====================================================================\n  # Loop through the training set nTrainRepetitions times\n  # ==========================================================================\n  for trainingNum in xrange(nTrainRepetitions):\n    if VERBOSITY >= 2:\n      print \"\\n##############################################################\"\n      print \"################# Training round #%d of %d #################\" \\\n                % (trainingNum, nTrainRepetitions)\n      for (name,tm) in tms.iteritems():\n        print \"TM parameters for %s: \" % (name)\n        print \"---------------------\"\n        tm.printParameters()\n        print\n\n    # ======================================================================\n    # Loop through the sequences in the training set\n    numSequences = len(testSequences)\n    for sequenceNum, trainingSequence in enumerate(trainingSequences):\n      numTimeSteps = len(trainingSequence)\n\n      if VERBOSITY >= 2:\n        print \"\\n================= Sequence #%d of %d ================\" \\\n                  % (sequenceNum, numSequences)\n\n      if doResets:\n        for tm in tms.itervalues():\n          tm.reset()\n\n      # --------------------------------------------------------------------\n      # Train each element of the sequence\n      for t, x in enumerate(trainingSequence):\n\n        # Print Verbose info about this element\n        if VERBOSITY >= 2:\n          print\n          if VERBOSITY >= 3:\n            print \"------------------------------------------------------------\"\n          print \"--------- sequence: #%d of %d, timeStep: #%d of %d -----------\" \\\n                  % (sequenceNum, numSequences, t, numTimeSteps)\n          firstTM.printInput(x)\n          print \"input nzs:\", x.nonzero()\n\n        # Train in this element\n        x = numpy.array(x).astype('float32')\n        for tm in tms.itervalues():\n          tm.learn(x, enableInference=True)\n\n        # Print the input and output states\n        if VERBOSITY >= 3:\n          for (name,tm) in tms.iteritems():\n            print \"I/O states of %s TM:\" % (name)\n            print \"-------------------------------------\",\n            tm.printStates(printPrevious = (VERBOSITY >= 5))\n            print\n\n        assertNoTMDiffs(tms)\n\n        # Print out number of columns that weren't predicted\n        if VERBOSITY >= 2:\n          for (name,tm) in tms.iteritems():\n            stats = tm.getStats()\n            print \"# of unpredicted columns for %s TM: %d of %d\" \\\n                % (name, stats['curMissing'], x.sum())\n            numBurstingCols = tm.infActiveState['t'].min(axis=1).sum()\n            print \"# of bursting columns for %s TM: %d of %d\" \\\n                % (name, numBurstingCols, x.sum())\n\n\n      # Print the trained cells\n      if VERBOSITY >= 4:\n        print \"Sequence %d finished.\" % (sequenceNum)\n        for (name,tm) in tms.iteritems():\n          print \"All cells of %s TM:\" % (name)\n          print \"-------------------------------------\",\n          tm.printCells()\n          print\n\n    # --------------------------------------------------------------------\n    # Done training all sequences in this round, print the total number of\n    #  missing, extra columns and make sure it's the same among the TMs\n    if VERBOSITY >= 2:\n      print\n    prevResult = None\n    for (name,tm) in tms.iteritems():\n      stats = tm.getStats()\n      if VERBOSITY >= 1:\n        print \"Stats for %s TM over all sequences for training round #%d of %d:\" \\\n                % (name, trainingNum, nTrainRepetitions)\n        print \"   total missing:\", stats['totalMissing']\n        print \"   total extra:\", stats['totalExtra']\n\n      if prevResult is None:\n        prevResult = (stats['totalMissing'], stats['totalExtra'])\n      else:\n        assert (stats['totalMissing'] == prevResult[0])\n        assert (stats['totalExtra'] == prevResult[1])\n\n      tm.resetStats()\n\n\n  # =====================================================================\n  # Finish up learning\n  if VERBOSITY >= 3:\n    print \"Calling trim segments\"\n  prevResult = None\n  for tm in tms.itervalues():\n    nSegsRemoved, nSynsRemoved = tm.trimSegments()\n    if prevResult is None:\n      prevResult = (nSegsRemoved, nSynsRemoved)\n    else:\n      assert (nSegsRemoved == prevResult[0])\n      assert (nSynsRemoved == prevResult[1])\n\n  assertNoTMDiffs(tms)\n\n  if VERBOSITY >= 4:\n    print \"Training completed. Complete state:\"\n    for (name,tm) in tms.iteritems():\n      print \"%s:\" % (name)\n      tm.printCells()\n      print\n\n\n  # ==========================================================================\n  # Infer\n  # ==========================================================================\n  if VERBOSITY >= 2:\n    print \"\\n##############################################################\"\n    print \"########################## Inference #########################\"\n\n  # Reset stats in all TMs\n  for tm in tms.itervalues():\n    tm.resetStats()\n\n  # -------------------------------------------------------------------\n  # Loop through the test sequences\n  numSequences = len(testSequences)\n  for sequenceNum, testSequence in enumerate(testSequences):\n    numTimeSteps = len(testSequence)\n\n    # Identify this sequence\n    if VERBOSITY >= 2:\n      print \"\\n================= Sequence %d of %d ================\" \\\n                % (sequenceNum, numSequences)\n\n    # Send in the rest\n    if doResets:\n      for tm in tms.itervalues():\n        tm.reset()\n\n    # -------------------------------------------------------------------\n    # Loop through the elements of this sequence\n    for t,x in enumerate(testSequence):\n\n      # Print verbose info about this element\n      if VERBOSITY >= 2:\n        print\n        if VERBOSITY >= 3:\n          print \"------------------------------------------------------------\"\n        print \"--------- sequence: #%d of %d, timeStep: #%d of %d -----------\" \\\n                % (sequenceNum, numSequences, t, numTimeSteps)\n        firstTM.printInput(x)\n        print \"input nzs:\", x.nonzero()\n\n      # Infer on this element\n      for tm in tms.itervalues():\n        tm.infer(x)\n\n      assertNoTMDiffs(tms)\n\n      # Print out number of columns that weren't predicted\n      if VERBOSITY >= 2:\n        for (name,tm) in tms.iteritems():\n          stats = tm.getStats()\n          print \"# of unpredicted columns for %s TM: %d of %d\" \\\n              % (name, stats['curMissing'], x.sum())\n\n      # Debug print of internal state\n      if VERBOSITY >= 3:\n        for (name,tm) in tms.iteritems():\n          print \"I/O states of %s TM:\" % (name)\n          print \"-------------------------------------\",\n          tm.printStates(printPrevious = (VERBOSITY >= 5),\n                         printLearnState = False)\n          print\n\n    # Done with this sequence\n    # Debug print of all stats of the TMs\n    if VERBOSITY >= 4:\n      print\n      for (name,tm) in tms.iteritems():\n        print \"Interim internal stats for %s TM:\" % (name)\n        print \"---------------------------------\"\n        pprint.pprint(tm.getStats())\n        print\n\n\n  if VERBOSITY >= 2:\n    print \"\\n##############################################################\"\n    print \"####################### Inference Done #######################\"\n\n  # Get the overall stats for each TM and return them\n  tmStats = dict()\n  for (name,tm) in tms.iteritems():\n    tmStats[name] = stats = tm.getStats()\n    if VERBOSITY >= 2:\n      print \"Stats for %s TM over all sequences:\" % (name)\n      print \"   total missing:\", stats['totalMissing']\n      print \"   total extra:\", stats['totalExtra']\n\n  for (name,tm) in tms.iteritems():\n    if VERBOSITY >= 3:\n      print \"\\nAll internal stats for %s TM:\" % (name)\n      print \"-------------------------------------\",\n      pprint.pprint(tmStats[name])\n      print\n\n  return tmStats"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndecide whether a particular character needs to be quoted.", "response": "def needsquoting(c, quotetabs, header):\n    \"\"\"Decide whether a particular character needs to be quoted.\n\n    The 'quotetabs' flag indicates whether embedded tabs and spaces should be\n    quoted.  Note that line-ending tabs and spaces are always encoded, as per\n    RFC 1521.\n    \"\"\"\n    if c in ' \\t':\n        return quotetabs\n    # if header, we have to escape _ because _ is used to escape space\n    if c == '_':\n        return header\n    return c == ESCAPE or not (' ' <= c <= '~')"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nquotes a single character.", "response": "def quote(c):\n    \"\"\"Quote a single character.\"\"\"\n    i = ord(c)\n    return ESCAPE + HEX[i//16] + HEX[i%16]"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nread input apply quoted - printable encoding and write to output.", "response": "def encode(input, output, quotetabs, header = 0):\n    \"\"\"Read 'input', apply quoted-printable encoding, and write to 'output'.\n\n    'input' and 'output' are files with readline() and write() methods.\n    The 'quotetabs' flag indicates whether embedded tabs and spaces should be\n    quoted.  Note that line-ending tabs and spaces are always encoded, as per\n    RFC 1521.\n    The 'header' flag indicates whether we are encoding spaces as _ as per\n    RFC 1522.\n    \"\"\"\n\n    if b2a_qp is not None:\n        data = input.read()\n        odata = b2a_qp(data, quotetabs = quotetabs, header = header)\n        output.write(odata)\n        return\n\n    def write(s, output=output, lineEnd='\\n'):\n        # RFC 1521 requires that the line ending in a space or tab must have\n        # that trailing character encoded.\n        if s and s[-1:] in ' \\t':\n            output.write(s[:-1] + quote(s[-1]) + lineEnd)\n        elif s == '.':\n            output.write(quote(s) + lineEnd)\n        else:\n            output.write(s + lineEnd)\n\n    prevline = None\n    while 1:\n        line = input.readline()\n        if not line:\n            break\n        outline = []\n        # Strip off any readline induced trailing newline\n        stripped = ''\n        if line[-1:] == '\\n':\n            line = line[:-1]\n            stripped = '\\n'\n        # Calculate the un-length-limited encoded line\n        for c in line:\n            if needsquoting(c, quotetabs, header):\n                c = quote(c)\n            if header and c == ' ':\n                outline.append('_')\n            else:\n                outline.append(c)\n        # First, write out the previous line\n        if prevline is not None:\n            write(prevline)\n        # Now see if we need any soft line breaks because of RFC-imposed\n        # length limitations.  Then do the thisline->prevline dance.\n        thisline = EMPTYSTRING.join(outline)\n        while len(thisline) > MAXLINESIZE:\n            # Don't forget to include the soft line break `=' sign in the\n            # length calculation!\n            write(thisline[:MAXLINESIZE-1], lineEnd='=\\n')\n            thisline = thisline[MAXLINESIZE-1:]\n        # Write out the current line\n        prevline = thisline\n    # Write out the last line, without a trailing newline\n    if prevline is not None:\n        write(prevline, lineEnd=stripped)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef decode(input, output, header = 0):\n\n    if a2b_qp is not None:\n        data = input.read()\n        odata = a2b_qp(data, header = header)\n        output.write(odata)\n        return\n\n    new = ''\n    while 1:\n        line = input.readline()\n        if not line: break\n        i, n = 0, len(line)\n        if n > 0 and line[n-1] == '\\n':\n            partial = 0; n = n-1\n            # Strip trailing whitespace\n            while n > 0 and line[n-1] in \" \\t\\r\":\n                n = n-1\n        else:\n            partial = 1\n        while i < n:\n            c = line[i]\n            if c == '_' and header:\n                new = new + ' '; i = i+1\n            elif c != ESCAPE:\n                new = new + c; i = i+1\n            elif i+1 == n and not partial:\n                partial = 1; break\n            elif i+1 < n and line[i+1] == ESCAPE:\n                new = new + ESCAPE; i = i+2\n            elif i+2 < n and ishex(line[i+1]) and ishex(line[i+2]):\n                new = new + chr(unhex(line[i+1:i+3])); i = i+3\n            else: # Bad escape sequence -- leave it in\n                new = new + c; i = i+1\n        if not partial:\n            output.write(new + '\\n')\n            new = ''\n    if new:\n        output.write(new)", "response": "Read input apply quoted - printable decoding and write to output."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef unhex(s):\n    bits = 0\n    for c in s:\n        if '0' <= c <= '9':\n            i = ord('0')\n        elif 'a' <= c <= 'f':\n            i = ord('a')-10\n        elif 'A' <= c <= 'F':\n            i = ord('A')-10\n        else:\n            break\n        bits = bits*16 + (ord(c) - i)\n    return bits", "response": "Get the integer value of a hexadecimal number."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nencoding a string using Base64.", "response": "def b64encode(s, altchars=None):\n    \"\"\"Encode a string using Base64.\n\n    s is the string to encode.  Optional altchars must be a string of at least\n    length 2 (additional characters are ignored) which specifies an\n    alternative alphabet for the '+' and '/' characters.  This allows an\n    application to e.g. generate url or filesystem safe Base64 strings.\n\n    The encoded string is returned.\n    \"\"\"\n    # Strip off the trailing newline\n    encoded = binascii.b2a_base64(s)[:-1]\n    if altchars is not None:\n        return encoded.translate(string.maketrans(b'+/', altchars[:2]))\n    return encoded"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef b64decode(s, altchars=None):\n    if altchars is not None:\n        s = s.translate(string.maketrans(altchars[:2], '+/'))\n    try:\n        return binascii.a2b_base64(s)\n    except binascii.Error, msg:\n        # Transform this exception for consistency\n        raise TypeError(msg)", "response": "Decode a Base64 encoded string."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nencode a string using Base32.", "response": "def b32encode(s):\n    \"\"\"Encode a string using Base32.\n\n    s is the string to encode.  The encoded string is returned.\n    \"\"\"\n    parts = []\n    quanta, leftover = divmod(len(s), 5)\n    # Pad the last quantum with zero bits if necessary\n    if leftover:\n        s += ('\\0' * (5 - leftover))\n        quanta += 1\n    for i in range(quanta):\n        # c1 and c2 are 16 bits wide, c3 is 8 bits wide.  The intent of this\n        # code is to process the 40 bits in units of 5 bits.  So we take the 1\n        # leftover bit of c1 and tack it onto c2.  Then we take the 2 leftover\n        # bits of c2 and tack them onto c3.  The shifts and masks are intended\n        # to give us values of exactly 5 bits in width.\n        c1, c2, c3 = struct.unpack('!HHB', s[i*5:(i+1)*5])\n        c2 += (c1 & 1) << 16 # 17 bits wide\n        c3 += (c2 & 3) << 8  # 10 bits wide\n        parts.extend([_b32tab[c1 >> 11],         # bits 1 - 5\n                      _b32tab[(c1 >> 6) & 0x1f], # bits 6 - 10\n                      _b32tab[(c1 >> 1) & 0x1f], # bits 11 - 15\n                      _b32tab[c2 >> 12],         # bits 16 - 20 (1 - 5)\n                      _b32tab[(c2 >> 7) & 0x1f], # bits 21 - 25 (6 - 10)\n                      _b32tab[(c2 >> 2) & 0x1f], # bits 26 - 30 (11 - 15)\n                      _b32tab[c3 >> 5],          # bits 31 - 35 (1 - 5)\n                      _b32tab[c3 & 0x1f],        # bits 36 - 40 (1 - 5)\n                      ])\n    encoded = EMPTYSTRING.join(parts)\n    # Adjust for any leftover partial quanta\n    if leftover == 1:\n        return encoded[:-6] + '======'\n    elif leftover == 2:\n        return encoded[:-4] + '===='\n    elif leftover == 3:\n        return encoded[:-3] + '==='\n    elif leftover == 4:\n        return encoded[:-1] + '='\n    return encoded"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef b32decode(s, casefold=False, map01=None):\n    quanta, leftover = divmod(len(s), 8)\n    if leftover:\n        raise TypeError('Incorrect padding')\n    # Handle section 2.4 zero and one mapping.  The flag map01 will be either\n    # False, or the character to map the digit 1 (one) to.  It should be\n    # either L (el) or I (eye).\n    if map01:\n        s = s.translate(string.maketrans(b'01', b'O' + map01))\n    if casefold:\n        s = s.upper()\n    # Strip off pad characters from the right.  We need to count the pad\n    # characters because this will tell us how many null bytes to remove from\n    # the end of the decoded string.\n    padchars = 0\n    mo = re.search('(?P<pad>[=]*)$', s)\n    if mo:\n        padchars = len(mo.group('pad'))\n        if padchars > 0:\n            s = s[:-padchars]\n    # Now decode the full quanta\n    parts = []\n    acc = 0\n    shift = 35\n    for c in s:\n        val = _b32rev.get(c)\n        if val is None:\n            raise TypeError('Non-base32 digit found')\n        acc += _b32rev[c] << shift\n        shift -= 5\n        if shift < 0:\n            parts.append(binascii.unhexlify('%010x' % acc))\n            acc = 0\n            shift = 35\n    # Process the last, partial quanta\n    last = binascii.unhexlify('%010x' % acc)\n    if padchars == 0:\n        last = ''                       # No characters\n    elif padchars == 1:\n        last = last[:-1]\n    elif padchars == 3:\n        last = last[:-2]\n    elif padchars == 4:\n        last = last[:-3]\n    elif padchars == 6:\n        last = last[:-4]\n    else:\n        raise TypeError('Incorrect padding')\n    parts.append(last)\n    return EMPTYSTRING.join(parts)", "response": "Decode a Base32 encoded string."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndecodes a Base16 encoded string.", "response": "def b16decode(s, casefold=False):\n    \"\"\"Decode a Base16 encoded string.\n\n    s is the string to decode.  Optional casefold is a flag specifying whether\n    a lowercase alphabet is acceptable as input.  For security purposes, the\n    default is False.\n\n    The decoded string is returned.  A TypeError is raised if s is\n    incorrectly padded or if there are non-alphabet characters present in the\n    string.\n    \"\"\"\n    if casefold:\n        s = s.upper()\n    if re.search('[^0-9A-F]', s):\n        raise TypeError('Non-base16 digit found')\n    return binascii.unhexlify(s)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nencode a string into multiple lines of base - 64 data.", "response": "def encodestring(s):\n    \"\"\"Encode a string into multiple lines of base-64 data.\"\"\"\n    pieces = []\n    for i in range(0, len(s), MAXBINSIZE):\n        chunk = s[i : i + MAXBINSIZE]\n        pieces.append(binascii.b2a_base64(chunk))\n    return \"\".join(pieces)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the source line from the source file at the given line number.", "response": "def source_line(self, lineno):\n        \"\"\"\n        Returns line ``lineno`` from source, taking ``first_line`` into account,\n        or raises :exc:`IndexError` if ``lineno`` is out of range.\n        \"\"\"\n        line_begins = self._extract_line_begins()\n        lineno = lineno - self.first_line\n        if lineno >= 0 and lineno + 1 < len(line_begins):\n            first, last = line_begins[lineno:lineno + 2]\n            return self.source[first:last]\n        elif lineno >= 0 and lineno < len(line_begins):\n            return self.source[line_begins[-1]:]\n        else:\n            raise IndexError"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef decompose_position(self, offset):\n        line_begins = self._extract_line_begins()\n        lineno = bisect.bisect_right(line_begins, offset) - 1\n        if offset >= 0 and offset <= len(self.source):\n            return lineno + self.first_line, offset - line_begins[lineno]\n        else:\n            raise IndexError", "response": "Decomposes a character offset into a line column tuple."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef chain(self, expanded_from):\n        return Range(self.source_buffer, self.begin_pos, self.begin_pos,\n                     expanded_from=expanded_from)", "response": "Returns a range identical to this one but indicating that that\n        it was expanded from the range expanded_from."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef begin(self):\n        return Range(self.source_buffer, self.begin_pos, self.begin_pos,\n                     expanded_from=self.expanded_from)", "response": "Returns a zero - length range located just before the beginning of this range."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a zero - length range located just after the end of this range.", "response": "def end(self):\n        \"\"\"\n        Returns a zero-length range located just after the end of this range.\n        \"\"\"\n        return Range(self.source_buffer, self.end_pos, self.end_pos,\n                     expanded_from=self.expanded_from)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the column number of the beginning of this range.", "response": "def column(self):\n        \"\"\"\n        Returns a zero-based column number of the beginning of this range.\n        \"\"\"\n        line, column = self.source_buffer.decompose_position(self.begin_pos)\n        return column"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a [ begin end ) tuple describing the range of columns spanned by this range.", "response": "def column_range(self):\n        \"\"\"\n        Returns a [*begin*, *end*) tuple describing the range of columns spanned\n        by this range. If range spans more than one line, returned *end* is\n        the last column of the line.\n        \"\"\"\n        if self.begin().line() == self.end().line():\n            return self.begin().column(), self.end().column()\n        else:\n            return self.begin().column(), len(self.begin().source_line()) - 1"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef line(self):\n        line, column = self.source_buffer.decompose_position(self.begin_pos)\n        return line", "response": "Returns the line number of the beginning of this range."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the smallest possible range spanning both this range and other.", "response": "def join(self, other):\n        \"\"\"\n        Returns the smallest possible range spanning both this range and other.\n        Raises :exc:`ValueError` if the ranges do not belong to the same\n        :class:`Buffer`.\n        \"\"\"\n        if self.source_buffer != other.source_buffer:\n            raise ValueError\n        if self.expanded_from == other.expanded_from:\n            expanded_from = self.expanded_from\n        else:\n            expanded_from = None\n        return Range(self.source_buffer,\n                     min(self.begin_pos, other.begin_pos),\n                     max(self.end_pos, other.end_pos),\n                     expanded_from=expanded_from)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef source_lines(self):\n        return [self.source_buffer.source_line(line)\n                for line in range(self.line(), self.end().line() + 1)]", "response": "Returns the source lines of the entirety of this range."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef rewrite(self):\n        self._sort()\n        self._check()\n\n        rewritten, pos = [], 0\n        for range, replacement in self.ranges:\n            rewritten.append(self.buffer.source[pos:range.begin_pos])\n            rewritten.append(replacement)\n            pos = range.end_pos\n        rewritten.append(self.buffer.source[pos:])\n\n        return Buffer(\"\".join(rewritten), self.buffer.name, self.buffer.first_line)", "response": "Return the rewritten source. May raise : class:`RewriterConflict`."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef compare(left, right, compare_locs=False):\n    if type(left) != type(right):\n        return False\n\n    if isinstance(left, ast.AST):\n        for field in left._fields:\n            if not compare(getattr(left, field), getattr(right, field)):\n                return False\n\n        if compare_locs:\n            for loc in left._locs:\n                if getattr(left, loc) != getattr(right, loc):\n                    return False\n\n        return True\n    elif isinstance(left, list):\n        if len(left) != len(right):\n            return False\n\n        for left_elt, right_elt in zip(left, right):\n            if not compare(left_elt, right_elt):\n                return False\n\n        return True\n    else:\n        return left == right", "response": "A comparison function that compares two objects."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef visit(self, obj):\n        if isinstance(obj, list):\n            return [self.visit(elt) for elt in obj]\n        elif isinstance(obj, ast.AST):\n            return self._visit_one(obj)", "response": "Visit a node or a list of nodes. Other values are ignored."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef generic_visit(self, node):\n        for field_name in node._fields:\n            setattr(node, field_name, self.visit(getattr(node, field_name)))\n        return node", "response": "Called by the visitor function to visit the node."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nvisit a node or a list of nodes. Other values are ignored.", "response": "def visit(self, obj):\n        \"\"\"Visit a node or a list of nodes. Other values are ignored\"\"\"\n        if isinstance(obj, list):\n            return list(filter(lambda x: x is not None, map(self.visit, obj)))\n        elif isinstance(obj, ast.AST):\n            return self._visit_one(obj)\n        else:\n            return obj"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef acquire(self, waitflag=None):\n        if waitflag is None or waitflag:\n            self.locked_status = True\n            return True\n        else:\n            if not self.locked_status:\n                self.locked_status = True\n                return True\n            else:\n                return False", "response": "This method is used to acquire a key from the cache."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef round_to_nearest(x):\n  int_part = int(x)\n  frac_part = x - int_part\n  if frac_part > 0.5 or frac_part == 0.5 and int_part & 1 == 1:\n    int_part += 1\n  return int_part", "response": "Python 3 style round a float x to the nearest int"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert a 32 - bit integer created by float_pack into a Python float.", "response": "def float_unpack(Q, size, le):\n  \"\"\"Convert a 32-bit or 64-bit integer created\n  by float_pack into a Python float.\"\"\"\n\n  if size == 8:\n    MIN_EXP = -1021  # = sys.float_info.min_exp\n    MAX_EXP = 1024   # = sys.float_info.max_exp\n    MANT_DIG = 53    # = sys.float_info.mant_dig\n    BITS = 64\n  elif size == 4:\n    MIN_EXP = -125   # C's FLT_MIN_EXP\n    MAX_EXP = 128    # FLT_MAX_EXP\n    MANT_DIG = 24    # FLT_MANT_DIG\n    BITS = 32\n  else:\n    raise ValueError(\"invalid size value\")\n\n  if Q >> BITS:\n    raise ValueError(\"input out of range\")\n\n  # extract pieces\n  sign = Q >> BITS - 1\n  exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1\n  mant = Q & ((1 << MANT_DIG - 1) - 1)\n\n  if exp == MAX_EXP - MIN_EXP + 2:\n    # nan or infinity\n    result = float('nan') if mant else float('inf')\n  elif exp == 0:\n    # subnormal or zero\n    result = math.ldexp(float(mant), MIN_EXP - MANT_DIG)\n  else:\n    # normal\n    mant += 1 << MANT_DIG - 1\n    result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG - 1)\n  return -result if sign else result"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef float_pack(x, size):\n\n  if size == 8:\n    MIN_EXP = -1021  # = sys.float_info.min_exp\n    MAX_EXP = 1024   # = sys.float_info.max_exp\n    MANT_DIG = 53    # = sys.float_info.mant_dig\n    BITS = 64\n  elif size == 4:\n    MIN_EXP = -125   # C's FLT_MIN_EXP\n    MAX_EXP = 128    # FLT_MAX_EXP\n    MANT_DIG = 24    # FLT_MANT_DIG\n    BITS = 32\n  else:\n    raise ValueError(\"invalid size value\")\n\n  sign = math.copysign(1.0, x) < 0.0\n  if math.isinf(x):\n    mant = 0\n    exp = MAX_EXP - MIN_EXP + 2\n  elif math.isnan(x):\n    mant = 1 << (MANT_DIG - 2)  # other values possible\n    exp = MAX_EXP - MIN_EXP + 2\n  elif x == 0.0:\n    mant = 0\n    exp = 0\n  else:\n    m, e = math.frexp(abs(x))  # abs(x) == m * 2**e\n    exp = e - (MIN_EXP - 1)\n    if exp > 0:\n      # Normal case.\n      mant = round_to_nearest(m * (1 << MANT_DIG))\n      mant -= 1 << MANT_DIG - 1\n    else:\n      # Subnormal case.\n      if exp + MANT_DIG - 1 >= 0:\n        mant = round_to_nearest(m * (1 << exp + MANT_DIG - 1))\n      else:\n        mant = 0\n      exp = 0\n\n    # Special case: rounding produced a MANT_DIG-bit mantissa.\n    assert 0 <= mant <= 1 << MANT_DIG - 1\n    if mant == 1 << MANT_DIG - 1:\n      mant = 0\n      exp += 1\n\n    # Raise on overflow (in some circumstances, may want to return\n    # infinity instead).\n    if exp >= MAX_EXP - MIN_EXP + 2:\n      raise OverflowError(\"float too large to pack in this format\")\n\n  # check constraints\n  assert 0 <= mant < 1 << MANT_DIG - 1\n  assert 0 <= exp <= MAX_EXP - MIN_EXP + 2\n  assert 0 <= sign <= 1\n  return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant", "response": "Convert a Python float x into a 64 - bit unsigned integer\n with the same byte representation."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef calcsize(fmt):\n\n  formatdef, endianness, i = getmode(fmt)\n  num = 0\n  result = 0\n  while i < len(fmt):\n    num, i = getNum(fmt, i)\n    cur = fmt[i]\n    try:\n      format = formatdef[cur]\n    except KeyError:\n      raise StructError(\"%s is not a valid format\" % cur)\n    if num != None:\n      result += num * format['size']\n    else:\n      result += format['size']\n    num = 0\n    i += 1\n  return result", "response": "Calculate size of a C struct described by format string fmt."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef pack(fmt, *args):\n  formatdef, endianness, i = getmode(fmt)\n  args = list(args)\n  n_args = len(args)\n  result = []\n  while i < len(fmt):\n    num, i = getNum(fmt, i)\n    cur = fmt[i]\n    try:\n      format = formatdef[cur]\n    except KeyError:\n      raise StructError(\"%s is not a valid format\" % cur)\n    if num == None:\n      num_s = 0\n      num = 1\n    else:\n      num_s = num\n\n    if cur == 'x':\n      result += [b'\\0' * num]\n    elif cur == 's':\n      if isinstance(args[0], bytes):\n        padding = num - len(args[0])\n        result += [args[0][:num] + b'\\0' * padding]\n        args.pop(0)\n      else:\n        raise StructError(\"arg for string format not a string\")\n    elif cur == 'p':\n      if isinstance(args[0], bytes):\n        padding = num - len(args[0]) - 1\n\n        if padding > 0:\n          result += [bytes([len(args[0])]) + args[0]\n                     [:num - 1] + b'\\0' * padding]\n        else:\n          if num < 255:\n            result += [bytes([num - 1]) + args[0][:num - 1]]\n          else:\n            result += [bytes([255]) + args[0][:num - 1]]\n        args.pop(0)\n      else:\n        raise StructError(\"arg for string format not a string\")\n\n    else:\n      if len(args) < num:\n        raise StructError(\"insufficient arguments to pack\")\n      for var in args[:num]:\n        result += [format['pack'](var, format['size'], endianness)]\n      args = args[num:]\n    num = None\n    i += 1\n  if len(args) != 0:\n    raise StructError(\"too many arguments for pack format\")\n  return b''.join(result)", "response": "pack returns a string containing values v1 v2... packed according to fmt."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nunpacking the string containing packed C structure data according to the format string fmt.", "response": "def unpack(fmt, data):\n  \"\"\"unpack(fmt, string) -> (v1, v2, ...)\n     Unpack the string, containing packed C structure data, according\n     to fmt.  Requires len(string)==calcsize(fmt).\n     See struct.__doc__ for more on format strings.\"\"\"\n  formatdef, endianness, i = getmode(fmt)\n  j = 0\n  num = 0\n  result = []\n  length = calcsize(fmt)\n  if length != len(data):\n    raise StructError(\"unpack str size does not match format\")\n  while i < len(fmt):\n    num, i = getNum(fmt, i)\n    cur = fmt[i]\n    i += 1\n    try:\n      format = formatdef[cur]\n    except KeyError:\n      raise StructError(\"%s is not a valid format\" % cur)\n\n    if not num:\n      num = 1\n\n    if cur == 'x':\n      j += num\n    elif cur == 's':\n      result.append(data[j:j + num])\n      j += num\n    elif cur == 'p':\n      n = data[j]\n      if n >= num:\n        n = num - 1\n      result.append(data[j + 1:j + n + 1])\n      j += num\n    else:\n      for n in range(num):\n        result += [format['unpack'](data, j, format['size'], endianness)]\n        j += format['size']\n\n  return tuple(result)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the human - readable representation of the diagnostic in the source file.", "response": "def render(self, only_line=False, colored=False):\n        \"\"\"\n        Returns the human-readable location of the diagnostic in the source,\n        the formatted message, the source line corresponding\n        to ``location`` and a line emphasizing the problematic\n        locations in the source line using ASCII art, as a list of lines.\n        Appends the result of calling :meth:`render` on ``notes``, if any.\n\n        For example: ::\n\n            <input>:1:8-9: error: cannot add integer and string\n            x + (1 + \"a\")\n                 ~ ^ ~~~\n\n        :param only_line: (bool) If true, only print line number, not line and column range\n        \"\"\"\n        source_line = self.location.source_line().rstrip(\"\\n\")\n        highlight_line = bytearray(re.sub(r\"[^\\t]\", \" \", source_line), \"utf-8\")\n\n        for hilight in self.highlights:\n            if hilight.line() == self.location.line():\n                lft, rgt = hilight.column_range()\n                highlight_line[lft:rgt] = bytearray(\"~\", \"utf-8\") * (rgt - lft)\n\n        lft, rgt = self.location.column_range()\n        if rgt == lft: # Expand zero-length ranges to one ^\n            rgt = lft + 1\n        highlight_line[lft:rgt] = bytearray(\"^\", \"utf-8\") * (rgt - lft)\n\n        if only_line:\n            location = \"%s:%s\" % (self.location.source_buffer.name, self.location.line())\n        else:\n            location = str(self.location)\n\n        notes = list(self.notes)\n        if self.level != \"note\":\n            expanded_location = self.location.expanded_from\n            while expanded_location is not None:\n                notes.insert(0, Diagnostic(\"note\",\n                    \"expanded from here\", {},\n                    self.location.expanded_from))\n                expanded_location = expanded_location.expanded_from\n\n        rendered_notes = reduce(list.__add__, [note.render(only_line, colored)\n                                               for note in notes], [])\n        if colored:\n            if self.level in (\"error\", \"fatal\"):\n                level_color = 31 # red\n            elif self.level == \"warning\":\n                level_color = 35 # magenta\n            else: # level == \"note\"\n                level_color = 30 # gray\n            return [\n                \"\\x1b[1;37m{}: \\x1b[{}m{}:\\x1b[37m {}\\x1b[0m\".\n                    format(location, level_color, self.level, self.message()),\n                source_line,\n                \"\\x1b[1;32m{}\\x1b[0m\".format(highlight_line.decode(\"utf-8\"))\n            ] + rendered_notes\n        else:\n            return [\n                \"{}: {}: {}\".format(location, self.level, self.message()),\n                source_line,\n                highlight_line.decode(\"utf-8\")\n            ] + rendered_notes"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef decode(input, output, encoding):\n    if encoding == 'base64':\n        import base64\n        return base64.decode(input, output)\n    if encoding == 'quoted-printable':\n        import quopri\n        return quopri.decode(input, output)\n    if encoding in ('uuencode', 'x-uuencode', 'uue', 'x-uue'):\n        import uu\n        return uu.decode(input, output)\n    if encoding in ('7bit', '8bit'):\n        return output.write(input.read())\n    if encoding in decodetab:\n        pipethrough(input, decodetab[encoding], output)\n    else:\n        raise ValueError, \\\n              'unknown Content-Transfer-Encoding: %s' % encoding", "response": "Decode common content - transfer - encodings."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef encode(input, output, encoding):\n    if encoding == 'base64':\n        import base64\n        return base64.encode(input, output)\n    if encoding == 'quoted-printable':\n        import quopri\n        return quopri.encode(input, output, 0)\n    if encoding in ('uuencode', 'x-uuencode', 'uue', 'x-uue'):\n        import uu\n        return uu.encode(input, output)\n    if encoding in ('7bit', '8bit'):\n        return output.write(input.read())\n    if encoding in encodetab:\n        pipethrough(input, encodetab[encoding], output)\n    else:\n        raise ValueError, \\\n              'unknown Content-Transfer-Encoding: %s' % encoding", "response": "Encode common content - transfer - encodings."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nprint the list of tuples as returned by extract_tb or CTYPE extract_stack as a formatted stack trace to the given file.", "response": "def print_list(extracted_list, file=None):\n    \"\"\"Print the list of tuples as returned by extract_tb() or\n    extract_stack() as a formatted stack trace to the given file.\"\"\"\n    if file is None:\n        file = sys.stderr\n    for filename, lineno, name, line in extracted_list:\n        _print(file,\n               '  File \"%s\", line %d, in %s' % (filename,lineno,name))\n        if line:\n            _print(file, '    %s' % line.strip())"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef format_list(extracted_list):\n    list = []\n    for filename, lineno, name, line in extracted_list:\n        item = '  File \"%s\", line %d, in %s\\n' % (filename,lineno,name)\n        if line:\n            item = item + '    %s\\n' % line.strip()\n        list.append(item)\n    return list", "response": "Format a list of tuples for printing."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nprinting up to limit stack trace entries from the traceback tb.", "response": "def print_tb(tb, limit=None, file=None):\n    \"\"\"Print up to 'limit' stack trace entries from the traceback 'tb'.\n\n    If 'limit' is omitted or None, all entries are printed.  If 'file'\n    is omitted or None, the output goes to sys.stderr; otherwise\n    'file' should be an open file or file-like object with a write()\n    method.\n    \"\"\"\n    if file is None:\n        file = sys.stderr\n    if limit is None:\n        if hasattr(sys, 'tracebacklimit'):\n            limit = sys.tracebacklimit\n    n = 0\n    while tb is not None and (limit is None or n < limit):\n        f = tb.tb_frame\n        lineno = tb.tb_lineno\n        co = f.f_code\n        filename = co.co_filename\n        name = co.co_name\n        _print(file,\n               '  File \"%s\", line %d, in %s' % (filename, lineno, name))\n        linecache.checkcache(filename)\n        line = linecache.getline(filename, lineno, f.f_globals)\n        if line: _print(file, '    ' + line.strip())\n        tb = tb.tb_next\n        n = n+1"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef print_exception(etype, value, tb, limit=None, file=None):\n    if file is None:\n        # TODO: Use sys.stderr when that's implemented.\n        file = open('/dev/stderr', 'w')\n        #file = sys.stderr\n    if tb:\n        _print(file, 'Traceback (most recent call last):')\n        print_tb(tb, limit, file)\n    lines = format_exception_only(etype, value)\n    for line in lines:\n        _print(file, line, '')", "response": "Print an exception to file."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nformats a stack trace and the exception information.", "response": "def format_exception(etype, value, tb, limit = None):\n    \"\"\"Format a stack trace and the exception information.\n\n    The arguments have the same meaning as the corresponding arguments\n    to print_exception().  The return value is a list of strings, each\n    ending in a newline and some containing internal newlines.  When\n    these lines are concatenated and printed, exactly the same text is\n    printed as does print_exception().\n    \"\"\"\n    if tb:\n        list = ['Traceback (most recent call last):\\n']\n        list = list + format_tb(tb, limit)\n    else:\n        list = []\n    list = list + format_exception_only(etype, value)\n    return list"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef format_exception_only(etype, value):\n\n    # An instance should not have a meaningful value parameter, but\n    # sometimes does, particularly for string exceptions, such as\n    # >>> raise string1, string2  # deprecated\n    #\n    # Clear these out first because issubtype(string1, SyntaxError)\n    # would raise another exception and mask the original problem.\n    if (isinstance(etype, BaseException) or\n#        isinstance(etype, types.InstanceType) or\n        etype is None or type(etype) is str):\n        return [_format_final_exc_line(etype, value)]\n\n    stype = etype.__name__\n\n    if not issubclass(etype, SyntaxError):\n        return [_format_final_exc_line(stype, value)]\n\n    # It was a syntax error; show exactly where the problem was found.\n    lines = []\n    try:\n        msg, (filename, lineno, offset, badline) = value.args\n    except Exception:\n        pass\n    else:\n        filename = filename or \"<string>\"\n        lines.append('  File \"%s\", line %d\\n' % (filename, lineno))\n        if badline is not None:\n            lines.append('    %s\\n' % badline.strip())\n            if offset is not None:\n                caretspace = badline.rstrip('\\n')\n                offset = min(len(caretspace), offset) - 1\n                caretspace = caretspace[:offset].lstrip()\n                # non-space whitespace (likes tabs) must be kept for alignment\n                caretspace = ((c.isspace() and c or ' ') for c in caretspace)\n                lines.append('    %s^\\n' % ''.join(caretspace))\n        value = msg\n\n    lines.append(_format_final_exc_line(stype, value))\n    return lines", "response": "Format an exception only exception in a list of strings."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a list of a single line for format_exception_only", "response": "def _format_final_exc_line(etype, value):\n    \"\"\"Return a list of a single line -- normal case for format_exception_only\"\"\"\n    valuestr = _some_str(value)\n    if value is None or not valuestr:\n        line = \"%s\\n\" % etype\n    else:\n        line = \"%s: %s\\n\" % (etype, valuestr)\n    return line"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef print_exc(limit=None, file=None):\n    if file is None:\n        # TODO: Use sys.stderr when that's implemented.\n        file = open('/dev/stderr', 'w')\n        #file = sys.stderr\n    try:\n        etype, value, tb = sys.exc_info()\n        print_exception(etype, value, tb, limit, file)\n    finally:\n        etype = value = tb = None", "response": "Print the current exception and all its sub - exceptions."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef print_stack(f=None, limit=None, file=None):\n    if f is None:\n        try:\n            raise ZeroDivisionError\n        except ZeroDivisionError:\n            f = sys.exc_info()[2].tb_frame.f_back\n    print_list(extract_stack(f, limit), file)", "response": "Print a stack trace from its invocation point."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef extract_stack(f=None, limit = None):\n    if f is None:\n        try:\n            raise ZeroDivisionError\n        except ZeroDivisionError:\n            f = sys.exc_info()[2].tb_frame.f_back\n    if limit is None:\n        if hasattr(sys, 'tracebacklimit'):\n            limit = sys.tracebacklimit\n    list = []\n    n = 0\n    while f is not None and (limit is None or n < limit):\n        lineno = f.f_lineno\n        co = f.f_code\n        filename = co.co_filename\n        name = co.co_name\n        linecache.checkcache(filename)\n        line = linecache.getline(filename, lineno, f.f_globals)\n        if line: line = line.strip()\n        else: line = None\n        list.append((filename, lineno, name, line))\n        f = f.f_back\n        n = n+1\n    list.reverse()\n    return list", "response": "Extract the raw traceback from the current stack frame."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize internal state of the random number generator.", "response": "def seed(self, a=None):\n        \"\"\"Initialize internal state of the random number generator.\n\n        None or no argument seeds from current time or from an operating\n        system specific randomness source if available.\n\n        If a is not None or is an int or long, hash(a) is used instead.\n        Hash values for some types are nondeterministic when the\n        PYTHONHASHSEED environment variable is enabled.\n        \"\"\"\n\n        super(Random, self).seed(a)\n        self.gauss_next = None"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef randrange(self, start, stop=None, step=1, _int=int, _maxwidth=1L<<BPF):\n\n        # This code is a bit messy to make it fast for the\n        # common case while still doing adequate error checking.\n        istart = _int(start)\n        if istart != start:\n            raise ValueError, \"non-integer arg 1 for randrange()\"\n        if stop is None:\n            if istart > 0:\n                if istart >= _maxwidth:\n                    return self._randbelow(istart)\n                return _int(self.random() * istart)\n            raise ValueError, \"empty range for randrange()\"\n\n        # stop argument supplied.\n        istop = _int(stop)\n        if istop != stop:\n            raise ValueError, \"non-integer stop for randrange()\"\n        width = istop - istart\n        if step == 1 and width > 0:\n            # Note that\n            #     int(istart + self.random()*width)\n            # instead would be incorrect.  For example, consider istart\n            # = -2 and istop = 0.  Then the guts would be in\n            # -2.0 to 0.0 exclusive on both ends (ignoring that random()\n            # might return 0.0), and because int() truncates toward 0, the\n            # final result would be -1 or 0 (instead of -2 or -1).\n            #     istart + int(self.random()*width)\n            # would also be incorrect, for a subtler reason:  the RHS\n            # can return a long, and then randrange() would also return\n            # a long, but we're supposed to return an int (for backward\n            # compatibility).\n\n            if width >= _maxwidth:\n                return _int(istart + self._randbelow(width))\n            return _int(istart + _int(self.random()*width))\n        if step == 1:\n            raise ValueError, \"empty range for randrange() (%d,%d, %d)\" % (istart, istop, width)\n\n        # Non-unit step argument supplied.\n        istep = _int(step)\n        if istep != step:\n            raise ValueError, \"non-integer step for randrange()\"\n        if istep > 0:\n            n = (width + istep - 1) // istep\n        elif istep < 0:\n            n = (width + istep + 1) // istep\n        else:\n            raise ValueError, \"zero step for randrange()\"\n\n        if n <= 0:\n            raise ValueError, \"empty range for randrange()\"\n\n        if n >= _maxwidth:\n            return istart + istep*self._randbelow(n)\n        return istart + istep*_int(self.random() * n)", "response": "Return a random item from the range start to stop."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nshuffles list x in place", "response": "def shuffle(self, x, random=None):\n        \"\"\"x, random=random.random -> shuffle list x in place; return None.\n\n        Optional arg random is a 0-argument function returning a random\n        float in [0.0, 1.0); by default, the standard random.random.\n\n        \"\"\"\n\n        if random is None:\n            random = self.random\n        _int = int\n        for i in reversed(xrange(1, len(x))):\n            # pick an element in x[:i+1] with which to exchange x[i]\n            j = _int(random() * (i+1))\n            x[i], x[j] = x[j], x[i]"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _slotnames(cls):\n\n    # Get the value from a cache in the class if possible\n    names = cls.__dict__.get(\"__slotnames__\")\n    if names is not None:\n        return names\n\n    # Not cached -- calculate the value\n    names = []\n    if not hasattr(cls, \"__slots__\"):\n        # This class has no slots\n        pass\n    else:\n        # Slots found -- gather slot names from all base classes\n        for c in cls.__mro__:\n            if \"__slots__\" in c.__dict__:\n                slots = c.__dict__['__slots__']\n                # if class has a single slot, it can be given as a string\n                if isinstance(slots, basestring):\n                    slots = (slots,)\n                for name in slots:\n                    # special descriptors\n                    if name in (\"__dict__\", \"__weakref__\"):\n                        continue\n                    # mangled names\n                    elif name.startswith('__') and not name.endswith('__'):\n                        names.append('_%s%s' % (c.__name__, name))\n                    else:\n                        names.append(name)\n\n    # Cache the outcome in the class if at all possible\n    try:\n        cls.__slotnames__ = names\n    except:\n        pass # But don't die if we can't\n\n    return names", "response": "Return a list of slot names for a given class."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add_extension(module, name, code):\n    code = int(code)\n    if not 1 <= code <= 0x7fffffff:\n        raise ValueError, \"code out of range\"\n    key = (module, name)\n    if (_extension_registry.get(key) == code and\n        _inverted_registry.get(code) == key):\n        return # Redundant registrations are benign\n    if key in _extension_registry:\n        raise ValueError(\"key %s is already registered with code %s\" %\n                         (key, _extension_registry[key]))\n    if code in _inverted_registry:\n        raise ValueError(\"code %s is already in use for key %s\" %\n                         (code, _inverted_registry[code]))\n    _extension_registry[key] = code\n    _inverted_registry[code] = key", "response": "Register an extension code."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef update_wrapper(wrapper,\n                   wrapped,\n                   assigned = WRAPPER_ASSIGNMENTS,\n                   updated = WRAPPER_UPDATES):\n    \"\"\"Update a wrapper function to look like the wrapped function\n\n       wrapper is the function to be updated\n       wrapped is the original function\n       assigned is a tuple naming the attributes assigned directly\n       from the wrapped function to the wrapper function (defaults to\n       functools.WRAPPER_ASSIGNMENTS)\n       updated is a tuple naming the attributes of the wrapper that\n       are updated with the corresponding attribute from the wrapped\n       function (defaults to functools.WRAPPER_UPDATES)\n    \"\"\"\n    for attr in assigned:\n        setattr(wrapper, attr, getattr(wrapped, attr))\n    for attr in updated:\n        getattr(wrapper, attr).update(getattr(wrapped, attr, {}))\n    # Return the wrapper so this can be used as a decorator via partial()\n    return wrapper", "response": "Update a wrapper function to look like the original function that is updated by the given parameters."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts a cmp = function into a key = function", "response": "def cmp_to_key(mycmp):\n    \"\"\"Convert a cmp= function into a key= function\"\"\"\n    class K(object):\n        __slots__ = ['obj']\n        def __init__(self, obj, *args):\n            self.obj = obj\n        def __lt__(self, other):\n            return mycmp(self.obj, other.obj) < 0\n        def __gt__(self, other):\n            return mycmp(self.obj, other.obj) > 0\n        def __eq__(self, other):\n            return mycmp(self.obj, other.obj) == 0\n        def __le__(self, other):\n            return mycmp(self.obj, other.obj) <= 0\n        def __ge__(self, other):\n            return mycmp(self.obj, other.obj) >= 0\n        def __ne__(self, other):\n            return mycmp(self.obj, other.obj) != 0\n        def __hash__(self):\n            raise TypeError('hash not implemented')\n    return K"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef unquote(s):\n    if len(s) > 1:\n        if s.startswith('\"') and s.endswith('\"'):\n            return s[1:-1].replace('\\\\\\\\', '\\\\').replace('\\\\\"', '\"')\n        if s.startswith('<') and s.endswith('>'):\n            return s[1:-1]\n    return s", "response": "Remove quotes from a string."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nparsing an address into a tuple.", "response": "def parseaddr(address):\n    \"\"\"Parse an address into a (realname, mailaddr) tuple.\"\"\"\n    a = AddressList(address)\n    lst = a.addresslist\n    if not lst:\n        return (None, None)\n    return lst[0]"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef mktime_tz(data):\n    if data[9] is None:\n        # No zone info, so localtime is better assumption than GMT\n        return time.mktime(data[:8] + (-1,))\n    else:\n        t = time.mktime(data[:8] + (0,))\n        return t - data[9] - time.timezone", "response": "Turn a 10 - tuple as returned by parsedate_tz() into a UTC timestamp."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef formatdate(timeval=None):\n    if timeval is None:\n        timeval = time.time()\n    timeval = time.gmtime(timeval)\n    return \"%s, %02d %s %04d %02d:%02d:%02d GMT\" % (\n            (\"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\")[timeval[6]],\n            timeval[2],\n            (\"Jan\", \"Feb\", \"Mar\", \"Apr\", \"May\", \"Jun\",\n             \"Jul\", \"Aug\", \"Sep\", \"Oct\", \"Nov\", \"Dec\")[timeval[1]-1],\n                                timeval[0], timeval[3], timeval[4], timeval[5])", "response": "Returns a date string for the given timeval."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrewind the file to the start of the body.", "response": "def rewindbody(self):\n        \"\"\"Rewind the file to the start of the body (if seekable).\"\"\"\n        if not self.seekable:\n            raise IOError, \"unseekable file\"\n        self.fp.seek(self.startofbody)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef readheaders(self):\n        self.dict = {}\n        self.unixfrom = ''\n        self.headers = lst = []\n        self.status = ''\n        headerseen = \"\"\n        firstline = 1\n        startofline = unread = tell = None\n        if hasattr(self.fp, 'unread'):\n            unread = self.fp.unread\n        elif self.seekable:\n            tell = self.fp.tell\n        while 1:\n            if tell:\n                try:\n                    startofline = tell()\n                except IOError:\n                    startofline = tell = None\n                    self.seekable = 0\n            line = self.fp.readline()\n            if not line:\n                self.status = 'EOF in headers'\n                break\n            # Skip unix From name time lines\n            if firstline and line.startswith('From '):\n                self.unixfrom = self.unixfrom + line\n                continue\n            firstline = 0\n            if headerseen and line[0] in ' \\t':\n                # It's a continuation line.\n                lst.append(line)\n                x = (self.dict[headerseen] + \"\\n \" + line.strip())\n                self.dict[headerseen] = x.strip()\n                continue\n            elif self.iscomment(line):\n                # It's a comment.  Ignore it.\n                continue\n            elif self.islast(line):\n                # Note! No pushback here!  The delimiter line gets eaten.\n                break\n            headerseen = self.isheader(line)\n            if headerseen:\n                # It's a legal header line, save it.\n                lst.append(line)\n                self.dict[headerseen] = line[len(headerseen)+1:].strip()\n                continue\n            elif headerseen is not None:\n                # An empty header name. These aren't allowed in HTTP, but it's\n                # probably a benign mistake. Don't add the header, just keep\n                # going.\n                continue\n            else:\n                # It's not a header line; throw it back and stop here.\n                if not self.dict:\n                    self.status = 'No headers'\n                else:\n                    self.status = 'Non-header line where header expected'\n                # Try to undo the read.\n                if unread:\n                    unread(line)\n                elif tell:\n                    self.fp.seek(startofline)\n                else:\n                    self.status = self.status + '; bad seek'\n                break", "response": "Reads the header lines from the file and returns a list of the header lines."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndetermine whether a given line is a legal header.", "response": "def isheader(self, line):\n        \"\"\"Determine whether a given line is a legal header.\n\n        This method should return the header name, suitably canonicalized.\n        You may override this method in order to use Message parsing on tagged\n        data in RFC 2822-like formats with special header formats.\n        \"\"\"\n        i = line.find(':')\n        if i > -1:\n            return line[:i].lower()\n        return None"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef getfirstmatchingheader(self, name):\n        name = name.lower() + ':'\n        n = len(name)\n        lst = []\n        hit = 0\n        for line in self.headers:\n            if hit:\n                if not line[:1].isspace():\n                    break\n            elif line[:n].lower() == name:\n                hit = 1\n            if hit:\n                lst.append(line)\n        return lst", "response": "Get the first matching header line matching name."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef getrawheader(self, name):\n\n        lst = self.getfirstmatchingheader(name)\n        if not lst:\n            return None\n        lst[0] = lst[0][len(name) + 1:]\n        return ''.join(lst)", "response": "A higher - level interface to getfirstmatchingheader."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the value of a given header name.", "response": "def getheader(self, name, default=None):\n        \"\"\"Get the header value for a name.\n\n        This is the normal interface: it returns a stripped version of the\n        header value for a given header name, or None if it doesn't exist.\n        This uses the dictionary version which finds the *last* such header.\n        \"\"\"\n        return self.dict.get(name.lower(), default)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets all values for a header. This returns a list of values for headers given more than once ; each value in the list is stripped in the same way as the result of getallmatchingheaders.", "response": "def getheaders(self, name):\n        \"\"\"Get all values for a header.\n\n        This returns a list of values for headers given more than once; each\n        value in the result list is stripped in the same way as the result of\n        getheader().  If the header is not given, return an empty list.\n        \"\"\"\n        result = []\n        current = ''\n        have_header = 0\n        for s in self.getallmatchingheaders(name):\n            if s[0].isspace():\n                if current:\n                    current = \"%s\\n %s\" % (current, s.strip())\n                else:\n                    current = s.strip()\n            else:\n                if have_header:\n                    result.append(current)\n                current = s[s.find(\":\") + 1:].strip()\n                have_header = 1\n        if have_header:\n            result.append(current)\n        return result"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget a list of addresses from a header.", "response": "def getaddrlist(self, name):\n        \"\"\"Get a list of addresses from a header.\n\n        Retrieves a list of addresses from a header, where each address is a\n        tuple as returned by getaddr().  Scans all named headers, so it works\n        properly with multiple To: or Cc: headers for example.\n        \"\"\"\n        raw = []\n        for h in self.getallmatchingheaders(name):\n            if h[0] in ' \\t':\n                raw.append(h)\n            else:\n                if raw:\n                    raw.append(', ')\n                i = h.find(':')\n                if i > 0:\n                    addr = h[i+1:]\n                raw.append(addr)\n        alladdrs = ''.join(raw)\n        a = AddressList(alladdrs)\n        return a.addresslist"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nparses up to the next address.", "response": "def gotonext(self):\n        \"\"\"Parse up to the start of the next address.\"\"\"\n        while self.pos < len(self.field):\n            if self.field[self.pos] in self.LWS + '\\n\\r':\n                self.pos = self.pos + 1\n            elif self.field[self.pos] == '(':\n                self.commentlist.append(self.getcomment())\n            else: break"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse all addresses. Returns a list containing all of the addresses.", "response": "def getaddrlist(self):\n        \"\"\"Parse all addresses.\n\n        Returns a list containing all of the addresses.\n        \"\"\"\n        result = []\n        ad = self.getaddress()\n        while ad:\n            result += ad\n            ad = self.getaddress()\n        return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getrouteaddr(self):\n        if self.field[self.pos] != '<':\n            return\n\n        expectroute = 0\n        self.pos += 1\n        self.gotonext()\n        adlist = \"\"\n        while self.pos < len(self.field):\n            if expectroute:\n                self.getdomain()\n                expectroute = 0\n            elif self.field[self.pos] == '>':\n                self.pos += 1\n                break\n            elif self.field[self.pos] == '@':\n                self.pos += 1\n                expectroute = 1\n            elif self.field[self.pos] == ':':\n                self.pos += 1\n            else:\n                adlist = self.getaddrspec()\n                self.pos += 1\n                break\n            self.gotonext()\n\n        return adlist", "response": "Parse a route address. This method just skips all the route stuff and returns the addrspec."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nparsing an RFC 2822 addr - spec.", "response": "def getaddrspec(self):\n        \"\"\"Parse an RFC 2822 addr-spec.\"\"\"\n        aslist = []\n\n        self.gotonext()\n        while self.pos < len(self.field):\n            if self.field[self.pos] == '.':\n                aslist.append('.')\n                self.pos += 1\n            elif self.field[self.pos] == '\"':\n                aslist.append('\"%s\"' % self.getquote())\n            elif self.field[self.pos] in self.atomends:\n                break\n            else: aslist.append(self.getatom())\n            self.gotonext()\n\n        if self.pos >= len(self.field) or self.field[self.pos] != '@':\n            return ''.join(aslist)\n\n        aslist.append('@')\n        self.pos += 1\n        self.gotonext()\n        return ''.join(aslist) + self.getdomain()"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the complete domain name from an address.", "response": "def getdomain(self):\n        \"\"\"Get the complete domain name from an address.\"\"\"\n        sdlist = []\n        while self.pos < len(self.field):\n            if self.field[self.pos] in self.LWS:\n                self.pos += 1\n            elif self.field[self.pos] == '(':\n                self.commentlist.append(self.getcomment())\n            elif self.field[self.pos] == '[':\n                sdlist.append(self.getdomainliteral())\n            elif self.field[self.pos] == '.':\n                self.pos += 1\n                sdlist.append('.')\n            elif self.field[self.pos] in self.atomends:\n                break\n            else: sdlist.append(self.getatom())\n        return ''.join(sdlist)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getdelimited(self, beginchar, endchars, allowcomments = 1):\n        if self.field[self.pos] != beginchar:\n            return ''\n\n        slist = ['']\n        quote = 0\n        self.pos += 1\n        while self.pos < len(self.field):\n            if quote == 1:\n                slist.append(self.field[self.pos])\n                quote = 0\n            elif self.field[self.pos] in endchars:\n                self.pos += 1\n                break\n            elif allowcomments and self.field[self.pos] == '(':\n                slist.append(self.getcomment())\n                continue        # have already advanced pos from getcomment\n            elif self.field[self.pos] == '\\\\':\n                quote = 1\n            else:\n                slist.append(self.field[self.pos])\n            self.pos += 1\n\n        return ''.join(slist)", "response": "Parse a header fragment delimited by special characters."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getphraselist(self):\n        plist = []\n\n        while self.pos < len(self.field):\n            if self.field[self.pos] in self.LWS:\n                self.pos += 1\n            elif self.field[self.pos] == '\"':\n                plist.append(self.getquote())\n            elif self.field[self.pos] == '(':\n                self.commentlist.append(self.getcomment())\n            elif self.field[self.pos] in self.phraseends:\n                break\n            else:\n                plist.append(self.getatom(self.phraseends))\n\n        return plist", "response": "Parse a sequence of RFC 2822 phrases."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nconstructing a date from a POSIX timestamp ( like time. time()).", "response": "def fromtimestamp(cls, t):\n        \"Construct a date from a POSIX timestamp (like time.time()).\"\n        y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t)\n        return cls(y, m, d)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns ctime () style string.", "response": "def ctime(self):\n        \"Return ctime() style string.\"\n        weekday = self.toordinal() % 7 or 7\n        return \"%s %s %2d 00:00:00 %04d\" % (\n            _DAYNAMES[weekday],\n            _MONTHNAMES[self._month],\n            self._day, self._year)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the date formatted according to ISO.", "response": "def isoformat(self):\n        \"\"\"Return the date formatted according to ISO.\n\n        This is 'YYYY-MM-DD'.\n\n        References:\n        - http://www.w3.org/TR/NOTE-datetime\n        - http://www.cl.cam.ac.uk/~mgk25/iso-time.html\n        \"\"\"\n        # return \"%04d-%02d-%02d\" % (self._year, self._month, self._day)\n        return \"%s-%s-%s\" % (str(self._year).zfill(4), str(self._month).zfill(2), str(self._day).zfill(2))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef replace(self, year=None, month=None, day=None):\n        if year is None:\n            year = self._year\n        if month is None:\n            month = self._month\n        if day is None:\n            day = self._day\n        return date.__new__(type(self), year, month, day)", "response": "Return a new date with new values for the specified fields."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a 3 - tuple containing the ISO year week number and weekday.", "response": "def isocalendar(self):\n        \"\"\"Return a 3-tuple containing ISO year, week number, and weekday.\n\n        The first ISO week of the year is the (Mon-Sun) week\n        containing the year's first Thursday; everything else derives\n        from that.\n\n        The first week is 1; Monday is 1 ... Sunday is 7.\n\n        ISO calendar algorithm taken from\n        http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm\n        \"\"\"\n        year = self._year\n        week1monday = _isoweek1monday(year)\n        today = _ymd2ord(self._year, self._month, self._day)\n        # Internally, week and day have origin 0\n        week, day = divmod(today - week1monday, 7)\n        if week < 0:\n            year -= 1\n            week1monday = _isoweek1monday(year)\n            week, day = divmod(today - week1monday, 7)\n        elif week >= 52:\n            if today >= _isoweek1monday(year+1):\n                year += 1\n                week = 0\n        return year, week+1, day+1"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _tzstr(self, sep=\":\"):\n        off = self._utcoffset()\n        if off is not None:\n            if off < 0:\n                sign = \"-\"\n                off = -off\n            else:\n                sign = \"+\"\n            hh, mm = divmod(off, 60)\n            assert 0 <= hh < 24\n            off = \"%s%02d%s%02d\" % (sign, hh, sep, mm)\n        return off", "response": "Return formatted timezone offset."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef isoformat(self):\n        s = _format_time(self._hour, self._minute, self._second,\n                         self._microsecond)\n        tz = self._tzstr()\n        if tz:\n            s += tz\n        return s", "response": "Return the time formatted according to ISO."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef tzname(self):\n        if self._tzinfo is None:\n            return None\n        name = self._tzinfo.tzname(None)\n        _check_tzname(name)\n        return name", "response": "Return the timezone name of the current object."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef dst(self):\n        if self._tzinfo is None:\n            return None\n        offset = self._tzinfo.dst(None)\n        offset = _check_utc_offset(\"dst\", offset)\n        if offset is not None:\n            offset = timedelta._create(0, offset * 60, 0, True)\n        return offset", "response": "Return the DST offset of the entry in the current timezone."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a new time with new values for the specified fields.", "response": "def replace(self, hour=None, minute=None, second=None, microsecond=None,\n                tzinfo=True):\n        \"\"\"Return a new time with new values for the specified fields.\"\"\"\n        if hour is None:\n            hour = self.hour\n        if minute is None:\n            minute = self.minute\n        if second is None:\n            second = self.second\n        if microsecond is None:\n            microsecond = self.microsecond\n        if tzinfo is True:\n            tzinfo = self.tzinfo\n        return time.__new__(type(self),\n                            hour, minute, second, microsecond, tzinfo)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nconstructs a datetime object from a POSIX timestamp.", "response": "def fromtimestamp(cls, timestamp, tz=None):\n        \"\"\"Construct a datetime from a POSIX timestamp (like time.time()).\n\n        A timezone info object may be passed in as well.\n        \"\"\"\n        _check_tzinfo_arg(tz)\n        converter = _time.localtime if tz is None else _time.gmtime\n        self = cls._from_timestamp(converter, timestamp, tz)\n        if tz is not None:\n            self = tz.fromutc(self)\n        return self"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconstruct a datetime from time. time() and optional time zone info.", "response": "def now(cls, tz=None):\n        \"Construct a datetime from time.time() and optional time zone info.\"\n        t = _time.time()\n        return cls.fromtimestamp(t, tz)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconstruct a datetime from a given date and a given time.", "response": "def combine(cls, date, time):\n        \"Construct a datetime from a given date and a given time.\"\n        if not isinstance(date, _date_class):\n            raise TypeError(\"date argument must be a date instance\")\n        if not isinstance(time, _time_class):\n            raise TypeError(\"time argument must be a time instance\")\n        return cls(date.year, date.month, date.day,\n                   time.hour, time.minute, time.second, time.microsecond,\n                   time.tzinfo)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef utctimetuple(self):\n        \"Return UTC time tuple compatible with time.gmtime().\"\n        y, m, d = self.year, self.month, self.day\n        hh, mm, ss = self.hour, self.minute, self.second\n        offset = self._utcoffset()\n        if offset:  # neither None nor 0\n            mm -= offset\n            y, m, d, hh, mm, ss, _ = _normalize_datetime(\n                y, m, d, hh, mm, ss, 0, ignore_overflow=True)\n        return _build_struct_time(y, m, d, hh, mm, ss, 0)", "response": "Return UTC time tuple compatible with time. gmtime()."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef time(self):\n        \"Return the time part, with tzinfo None.\"\n        return time(self.hour, self.minute, self.second, self.microsecond)", "response": "Return the time part with tzinfo None."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the time part with same tzinfo.", "response": "def timetz(self):\n        \"Return the time part, with same tzinfo.\"\n        return time(self.hour, self.minute, self.second, self.microsecond,\n                    self._tzinfo)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a new datetime with new values for the specified fields.", "response": "def replace(self, year=None, month=None, day=None, hour=None,\n                minute=None, second=None, microsecond=None, tzinfo=True):\n        \"\"\"Return a new datetime with new values for the specified fields.\"\"\"\n        if year is None:\n            year = self.year\n        if month is None:\n            month = self.month\n        if day is None:\n            day = self.day\n        if hour is None:\n            hour = self.hour\n        if minute is None:\n            minute = self.minute\n        if second is None:\n            second = self.second\n        if microsecond is None:\n            microsecond = self.microsecond\n        if tzinfo is True:\n            tzinfo = self.tzinfo\n        return datetime.__new__(type(self),\n                                year, month, day, hour, minute, second,\n                                microsecond, tzinfo)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ctime(self):\n        \"Return ctime() style string.\"\n        weekday = self.toordinal() % 7 or 7\n        return \"%s %s %2d %02d:%02d:%02d %04d\" % (\n            _DAYNAMES[weekday],\n            _MONTHNAMES[self._month],\n            self._day,\n            self._hour, self._minute, self._second,\n            self._year)", "response": "Return ctime () style string."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef isoformat(self, sep='T'):\n        s = (\"%04d-%02d-%02d%c\" % (self._year, self._month, self._day, sep) +\n             _format_time(self._hour, self._minute, self._second,\n                          self._microsecond))\n        off = self._utcoffset()\n        if off is not None:\n            if off < 0:\n                sign = \"-\"\n                off = -off\n            else:\n                sign = \"+\"\n            hh, mm = divmod(off, 60)\n            s += \"%s%02d:%02d\" % (sign, hh, mm)\n        return s", "response": "Return the time formatted according to ISO."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef utcoffset(self):\n        if self._tzinfo is None:\n            return None\n        offset = self._tzinfo.utcoffset(self)\n        offset = _check_utc_offset(\"utcoffset\", offset)\n        if offset is not None:\n            offset = timedelta._create(0, offset * 60, 0, True)\n        return offset", "response": "Return the timezone offset in minutes east of UTC ( negative west of\n        UTC"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the number of times b occurs in a.", "response": "def countOf(a, b):\n    \"Return the number of times b occurs in a.\"\n    count = 0\n    for i in a:\n        if i == b:\n            count += 1\n    return count"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef indexOf(a, b):\n    \"Return the first index of b in a.\"\n    for i, j in enumerate(a):\n        if j == b:\n            return i\n    else:\n        raise ValueError('sequence.index(x): x not in sequence')", "response": "Return the first index of b in a."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef length_hint(obj, default=0):\n    if not isinstance(default, int):\n        msg = (\"'%s' object cannot be interpreted as an integer\" %\n               type(default).__name__)\n        raise TypeError(msg)\n\n    try:\n        return len(obj)\n    except TypeError:\n        pass\n\n    try:\n        hint = type(obj).__length_hint__\n    except AttributeError:\n        return default\n\n    try:\n        val = hint(obj)\n    except TypeError:\n        return default\n    if val is NotImplemented:\n        return default\n    if not isinstance(val, int):\n        msg = ('__length_hint__ must be integer, not %s' %\n               type(val).__name__)\n        raise TypeError(msg)\n    if val < 0:\n        msg = '__length_hint__() should return >= 0'\n        raise ValueError(msg)\n    return val", "response": "Return an estimate of the number of items in obj."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef iconcat(a, b):\n    \"Same as a += b, for a and b sequences.\"\n    if not hasattr(a, '__getitem__'):\n        msg = \"'%s' object can't be concatenated\" % type(a).__name__\n        raise TypeError(msg)\n    a += b\n    return a", "response": "Same as a += b for a and b sequences."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef itruediv(a, b):\n    \"Same as a /= b.\"\n    if type(a) == int or type(a) == long:\n        a = float(a)\n    a /= b\n    return a", "response": "Same as a /= b."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a dialect corresponding to the sample and returns None if the sample is not a valid CSV file.", "response": "def sniff(self, sample, delimiters=None):\n        \"\"\"\n        Returns a dialect (or None) corresponding to the sample\n        \"\"\"\n\n        quotechar, doublequote, delimiter, skipinitialspace = \\\n                   self._guess_quote_and_delimiter(sample, delimiters)\n        if not delimiter:\n            delimiter, skipinitialspace = self._guess_delimiter(sample,\n                                                                delimiters)\n\n        if not delimiter:\n            raise Error, \"Could not determine delimiter\"\n\n        class dialect(Dialect):\n            _name = \"sniffed\"\n            lineterminator = '\\r\\n'\n            quoting = QUOTE_MINIMAL\n            # escapechar = ''\n\n        dialect.doublequote = doublequote\n        dialect.delimiter = delimiter\n        # _csv.reader won't accept a quotechar of ''\n        dialect.quotechar = quotechar or '\"'\n        dialect.skipinitialspace = skipinitialspace\n\n        return dialect"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nguess the quote and delimiter from the data.", "response": "def _guess_quote_and_delimiter(self, data, delimiters):\n        \"\"\"\n        Looks for text enclosed between two identical quotes\n        (the probable quotechar) which are preceded and followed\n        by the same character (the probable delimiter).\n        For example:\n                         ,'some text',\n        The quote with the most wins, same with the delimiter.\n        If there is no quotechar the delimiter can't be determined\n        this way.\n        \"\"\"\n\n        matches = []\n        for restr in ('(?P<delim>[^\\w\\n\"\\'])(?P<space> ?)(?P<quote>[\"\\']).*?(?P=quote)(?P=delim)', # ,\".*?\",\n                      '(?:^|\\n)(?P<quote>[\"\\']).*?(?P=quote)(?P<delim>[^\\w\\n\"\\'])(?P<space> ?)',   #  \".*?\",\n                      '(?P<delim>>[^\\w\\n\"\\'])(?P<space> ?)(?P<quote>[\"\\']).*?(?P=quote)(?:$|\\n)',  # ,\".*?\"\n                      '(?:^|\\n)(?P<quote>[\"\\']).*?(?P=quote)(?:$|\\n)'):                            #  \".*?\" (no delim, no space)\n            regexp = re.compile(restr, re.DOTALL | re.MULTILINE)\n            matches = regexp.findall(data)\n            if matches:\n                break\n\n        if not matches:\n            # (quotechar, doublequote, delimiter, skipinitialspace)\n            return ('', False, None, 0)\n        quotes = {}\n        delims = {}\n        spaces = 0\n        for m in matches:\n            n = regexp.groupindex['quote'] - 1\n            key = m[n]\n            if key:\n                quotes[key] = quotes.get(key, 0) + 1\n            try:\n                n = regexp.groupindex['delim'] - 1\n                key = m[n]\n            except KeyError:\n                continue\n            if key and (delimiters is None or key in delimiters):\n                delims[key] = delims.get(key, 0) + 1\n            try:\n                n = regexp.groupindex['space'] - 1\n            except KeyError:\n                continue\n            if m[n]:\n                spaces += 1\n\n        quotechar = reduce(lambda a, b, quotes = quotes:\n                           (quotes[a] > quotes[b]) and a or b, quotes.keys())\n\n        if delims:\n            delim = reduce(lambda a, b, delims = delims:\n                           (delims[a] > delims[b]) and a or b, delims.keys())\n            skipinitialspace = delims[delim] == spaces\n            if delim == '\\n': # most likely a file with a single column\n                delim = ''\n        else:\n            # there is *no* delimiter, it's a single column of quoted data\n            delim = ''\n            skipinitialspace = 0\n\n        # if we see an extra quote between delimiters, we've got a\n        # double quoted format\n        dq_regexp = re.compile(\n                               r\"((%(delim)s)|^)\\W*%(quote)s[^%(delim)s\\n]*%(quote)s[^%(delim)s\\n]*%(quote)s\\W*((%(delim)s)|$)\" % \\\n                               {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE)\n\n\n\n        if dq_regexp.search(data):\n            doublequote = True\n        else:\n            doublequote = False\n\n        return (quotechar, doublequote, delim, skipinitialspace)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nguesses the delimiter for the current canon entry.", "response": "def _guess_delimiter(self, data, delimiters):\n        \"\"\"\n        The delimiter /should/ occur the same number of times on\n        each row. However, due to malformed data, it may not. We don't want\n        an all or nothing approach, so we allow for small variations in this\n        number.\n          1) build a table of the frequency of each character on every line.\n          2) build a table of frequencies of this frequency (meta-frequency?),\n             e.g.  'x occurred 5 times in 10 rows, 6 times in 1000 rows,\n             7 times in 2 rows'\n          3) use the mode of the meta-frequency to determine the /expected/\n             frequency for that character\n          4) find out how often the character actually meets that goal\n          5) the character that best meets its goal is the delimiter\n        For performance reasons, the data is evaluated in chunks, so it can\n        try and evaluate the smallest portion of the data possible, evaluating\n        additional chunks as necessary.\n        \"\"\"\n\n        data = filter(None, data.split('\\n'))\n\n        ascii = [chr(c) for c in range(127)] # 7-bit ASCII\n\n        # build frequency tables\n        chunkLength = min(10, len(data))\n        iteration = 0\n        charFrequency = {}\n        modes = {}\n        delims = {}\n        start, end = 0, min(chunkLength, len(data))\n        while start < len(data):\n            iteration += 1\n            for line in data[start:end]:\n                for char in ascii:\n                    metaFrequency = charFrequency.get(char, {})\n                    # must count even if frequency is 0\n                    freq = line.count(char)\n                    # value is the mode\n                    metaFrequency[freq] = metaFrequency.get(freq, 0) + 1\n                    charFrequency[char] = metaFrequency\n\n            for char in charFrequency.keys():\n                items = charFrequency[char].items()\n                if len(items) == 1 and items[0][0] == 0:\n                    continue\n                # get the mode of the frequencies\n                if len(items) > 1:\n                    modes[char] = reduce(lambda a, b: a[1] > b[1] and a or b,\n                                         items)\n                    # adjust the mode - subtract the sum of all\n                    # other frequencies\n                    items.remove(modes[char])\n                    modes[char] = (modes[char][0], modes[char][1]\n                                   - reduce(lambda a, b: (0, a[1] + b[1]),\n                                            items)[1])\n                else:\n                    modes[char] = items[0]\n\n            # build a list of possible delimiters\n            modeList = modes.items()\n            total = float(chunkLength * iteration)\n            # (rows of consistent data) / (number of rows) = 100%\n            consistency = 1.0\n            # minimum consistency threshold\n            threshold = 0.9\n            while len(delims) == 0 and consistency >= threshold:\n                for k, v in modeList:\n                    if v[0] > 0 and v[1] > 0:\n                        if ((v[1]/total) >= consistency and\n                            (delimiters is None or k in delimiters)):\n                            delims[k] = v\n                consistency -= 0.01\n\n            if len(delims) == 1:\n                delim = delims.keys()[0]\n                skipinitialspace = (data[0].count(delim) ==\n                                    data[0].count(\"%c \" % delim))\n                return (delim, skipinitialspace)\n\n            # analyze another chunkLength lines\n            start = end\n            end += chunkLength\n\n        if not delims:\n            return ('', 0)\n\n        # if there's more than one, fall back to a 'preferred' list\n        if len(delims) > 1:\n            for d in self.preferred:\n                if d in delims.keys():\n                    skipinitialspace = (data[0].count(d) ==\n                                        data[0].count(\"%c \" % d))\n                    return (d, skipinitialspace)\n\n        # nothing else indicates a preference, pick the character that\n        # dominates(?)\n        items = [(v,k) for (k,v) in delims.items()]\n        items.sort()\n        delim = items[-1][1]\n\n        skipinitialspace = (data[0].count(delim) ==\n                            data[0].count(\"%c \" % delim))\n        return (delim, skipinitialspace)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef encode_basestring(s):\n    def replace(match):\n        return ESCAPE_DCT[match.group(0)]\n    return '\"' + ESCAPE.sub(replace, s) + '\"'", "response": "Return a JSON representation of a Python string"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef sub(pattern, repl, string, count=0, flags=0):\n    return _compile(pattern, flags).sub(repl, string, count)", "response": "Return the string obtained by replacing the leftmost\n    non - overlapping occurrences of the pattern in string by the repl."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a 2 - tuple containing the string obtained by replacing the leftmost non - overlapping occurrences of the pattern with the replacement repl.", "response": "def subn(pattern, repl, string, count=0, flags=0):\n    \"\"\"Return a 2-tuple containing (new_string, number).\n    new_string is the string obtained by replacing the leftmost\n    non-overlapping occurrences of the pattern in the source\n    string by the replacement repl.  number is the number of\n    substitutions that were made. repl can be either a string or a\n    callable; if a string, backslash escapes in it are processed.\n    If it is a callable, it's passed the match object and must\n    return a replacement string to be used.\"\"\"\n    return _compile(pattern, flags).subn(repl, string, count)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef split(pattern, string, maxsplit=0, flags=0):\n    return _compile(pattern, flags).split(string, maxsplit)", "response": "Splits the source string by the occurrences of the pattern and returns a list containing the resulting substrings."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a list of all non - overlapping matches in the string.", "response": "def findall(pattern, string, flags=0):\n    \"\"\"Return a list of all non-overlapping matches in the string.\n\n    If one or more groups are present in the pattern, return a\n    list of groups; this will be a list of tuples if the pattern\n    has more than one group.\n\n    Empty matches are included in the result.\"\"\"\n    return _compile(pattern, flags).findall(string)\n\n    # if sys.hexversion >= 0x02020000:\n    #     __all__.append(\"finditer\")\n    def finditer(pattern, string, flags=0):\n        \"\"\"Return an iterator over all non-overlapping matches in the\n        string.  For each match, the iterator returns a match object.\n\n        Empty matches are included in the result.\"\"\"\n        return _compile(pattern, flags).finditer(string)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nescapes all non - alphanumeric characters in pattern.", "response": "def escape(pattern):\n    \"Escape all non-alphanumeric characters in pattern.\"\n    s = list(pattern)\n    alphanum = _alphanum\n    for i, c in enumerate(pattern):\n        if c not in alphanum:\n            if c == \"\\000\":\n                s[i] = \"\\\\000\"\n            else:\n                s[i] = \"\\\\\" + c\n    return pattern[:0].join(s)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef alloc_temp(self, type_='*\u03c0g.Object'):\n    for v in sorted(self.free_temps, key=lambda k: k.name):\n      if v.type_ == type_:\n        self.free_temps.remove(v)\n        self.used_temps.add(v)\n        return v\n    self.temp_index += 1\n    name = '\u03c0Temp{:03d}'.format(self.temp_index)\n    v = expr.GeneratedTempVar(self, name, type_)\n    self.used_temps.add(v)\n    return v", "response": "Allocate a temporary Go variable having type type_ for this block."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreleases the GeneratedTempVar v so it can be reused.", "response": "def free_temp(self, v):\n    \"\"\"Release the GeneratedTempVar v so it can be reused.\"\"\"\n    self.used_temps.remove(v)\n    self.free_temps.add(v)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing a buffer and return an AST and comments as well.", "response": "def parse_buffer(buffer, mode=\"exec\", flags=[], version=None, engine=None):\n    \"\"\"\n    Like :meth:`parse`, but accepts a :class:`source.Buffer` instead of\n    source and filename, and returns comments as well.\n\n    :see: :meth:`parse`\n    :return: (:class:`ast.AST`, list of :class:`source.Comment`)\n        Abstract syntax tree and comments\n    \"\"\"\n\n    if version is None:\n        version = sys.version_info[0:2]\n\n    if engine is None:\n        engine = pythonparser_diagnostic.Engine()\n\n    lexer = pythonparser_lexer.Lexer(buffer, version, engine)\n    if mode in (\"single\", \"eval\"):\n        lexer.interactive = True\n\n    parser = pythonparser_parser.Parser(lexer, version, engine)\n    parser.add_flags(flags)\n\n    if mode == \"exec\":\n        return parser.file_input(), lexer.comments\n    elif mode == \"single\":\n        return parser.single_input(), lexer.comments\n    elif mode == \"eval\":\n        return parser.eval_input(), lexer.comments"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nparse a string into an abstract syntax tree.", "response": "def parse(source, filename=\"<unknown>\", mode=\"exec\",\n          flags=[], version=None, engine=None):\n    \"\"\"\n    Parse a string into an abstract syntax tree.\n    This is the replacement for the built-in :meth:`..ast.parse`.\n\n    :param source: (string) Source code in the correct encoding\n    :param filename: (string) Filename of the source (used in diagnostics)\n    :param mode: (string) Execution mode. Pass ``\"exec\"`` to parse a module,\n        ``\"single\"`` to parse a single (interactive) statement,\n        and ``\"eval\"`` to parse an expression. In the last two cases,\n        ``source`` must be terminated with an empty line\n        (i.e. end with ``\"\\\\n\\\\n\"``).\n    :param flags: (list of string) Future flags.\n        Equivalent to ``from __future__ import <flags>``.\n    :param version: (2-tuple of int) Major and minor version of Python\n        syntax to recognize, ``sys.version_info[0:2]`` by default.\n    :param engine: (:class:`diagnostic.Engine`) Diagnostic engine,\n        a fresh one is created by default\n    :return: (:class:`ast.AST`) Abstract syntax tree\n    :raise: :class:`diagnostic.Error`\n        if the source code is not well-formed\n    \"\"\"\n    ast, comments = parse_buffer(pythonparser_source.Buffer(source, filename),\n                                 mode, flags, version, engine)\n    return ast"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef encode(in_file, out_file, name=None, mode=None):\n    #\n    # If in_file is a pathname open it and change defaults\n    #\n    opened_files = []\n    try:\n        if in_file == '-':\n            in_file = sys.stdin\n        elif isinstance(in_file, basestring):\n            if name is None:\n                name = os.path.basename(in_file)\n            if mode is None:\n                try:\n                    mode = os.stat(in_file).st_mode\n                except AttributeError:\n                    pass\n            in_file = open(in_file, 'rb')\n            opened_files.append(in_file)\n        #\n        # Open out_file if it is a pathname\n        #\n        if out_file == '-':\n            out_file = sys.stdout\n        elif isinstance(out_file, basestring):\n            out_file = open(out_file, 'wb')\n            opened_files.append(out_file)\n        #\n        # Set defaults for name and mode\n        #\n        if name is None:\n            name = '-'\n        if mode is None:\n            mode = 0666\n        #\n        # Write the data\n        #\n        out_file.write('begin %o %s\\n' % ((mode&0777),name))\n        data = in_file.read(45)\n        while len(data) > 0:\n            out_file.write(binascii.b2a_uu(data))\n            data = in_file.read(45)\n        out_file.write(' \\nend\\n')\n    finally:\n        for f in opened_files:\n            f.close()", "response": "Encode a single uuencoded file."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndecoding a uuencoded file into a single file.", "response": "def decode(in_file, out_file=None, mode=None, quiet=0):\n    \"\"\"Decode uuencoded file\"\"\"\n    #\n    # Open the input file, if needed.\n    #\n    opened_files = []\n    if in_file == '-':\n        in_file = sys.stdin\n    elif isinstance(in_file, basestring):\n        in_file = open(in_file)\n        opened_files.append(in_file)\n    try:\n        #\n        # Read until a begin is encountered or we've exhausted the file\n        #\n        while True:\n            hdr = in_file.readline()\n            if not hdr:\n                raise Error('No valid begin line found in input file')\n            if not hdr.startswith('begin'):\n                continue\n            hdrfields = hdr.split(' ', 2)\n            if len(hdrfields) == 3 and hdrfields[0] == 'begin':\n                try:\n                    int(hdrfields[1], 8)\n                    break\n                except ValueError:\n                    pass\n        if out_file is None:\n            out_file = hdrfields[2].rstrip()\n            if os.path.exists(out_file):\n                raise Error('Cannot overwrite existing file: %s' % out_file)\n        if mode is None:\n            mode = int(hdrfields[1], 8)\n        #\n        # Open the output file\n        #\n        if out_file == '-':\n            out_file = sys.stdout\n        elif isinstance(out_file, basestring):\n            fp = open(out_file, 'wb')\n            try:\n                os.path.chmod(out_file, mode)\n            except AttributeError:\n                pass\n            out_file = fp\n            opened_files.append(out_file)\n        #\n        # Main decoding loop\n        #\n        s = in_file.readline()\n        while s and s.strip() != 'end':\n            try:\n                data = binascii.a2b_uu(s)\n            except binascii.Error, v:\n                # Workaround for broken uuencoders by /Fredrik Lundh\n                nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3\n                data = binascii.a2b_uu(s[:nbytes])\n                if not quiet:\n                    sys.stderr.write(\"Warning: %s\\n\" % v)\n            out_file.write(data)\n            s = in_file.readline()\n        if not s:\n            raise Error('Truncated input file')\n    finally:\n        for f in opened_files:\n            f.close()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsplit the argument into words using split capitalize each word using capitalize and join the capitalized words using join.", "response": "def capwords(s, sep=None):\n    \"\"\"capwords(s [,sep]) -> string\n\n    Split the argument into words using split, capitalize each\n    word using capitalize, and join the capitalized words using\n    join.  If the optional second argument sep is absent or None,\n    runs of whitespace characters are replaced by a single space\n    and leading and trailing whitespace are removed, otherwise\n    sep is used to split and join the words.\n\n    \"\"\"\n    return (sep or ' ').join(x.capitalize() for x in s.split(sep))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef maketrans(fromstr, tostr):\n    if len(fromstr) != len(tostr):\n        raise ValueError, \"maketrans arguments must have same length\"\n    global _idmapL\n    if not _idmapL:\n        _idmapL = list(_idmap)\n    L = _idmapL[:]\n    fromstr = map(ord, fromstr)\n    for i in range(len(fromstr)):\n        L[fromstr[i]] = tostr[i]\n    return ''.join(L)", "response": "Create a translation table from two strings."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a string with zeros on the left of x with the specified width.", "response": "def zfill(x, width):\n    \"\"\"zfill(x, width) -> string\n\n    Pad a numeric string x with zeros on the left, to fill a field\n    of the specified width.  The string x is never truncated.\n\n    \"\"\"\n    if not isinstance(x, basestring):\n        x = repr(x)\n    return x.zfill(width)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef translate(s, table, deletions=\"\"):\n    if deletions or table is None:\n        return s.translate(table, deletions)\n    else:\n        # Add s[:0] so that if s is Unicode and table is an 8-bit string,\n        # table is converted to Unicode.  This means that table *cannot*\n        # be a dictionary -- for that feature, use u.translate() directly.\n        return s.translate(table + s[:0])", "response": "Translate a string s to a Unicode string s."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a copy of string s with all occurrences of old replaced by new.", "response": "def replace(s, old, new, maxreplace=-1):\n    \"\"\"replace (str, old, new[, maxreplace]) -> string\n\n    Return a copy of string str with all occurrences of substring\n    old replaced by new. If the optional argument maxreplace is\n    given, only the first maxreplace occurrences are replaced.\n\n    \"\"\"\n    return s.replace(old, new, maxreplace)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nusing SequenceMatcher to return list of the best good enough matches for word and list of possibilities.", "response": "def get_close_matches(word, possibilities, n=3, cutoff=0.6):\n    \"\"\"Use SequenceMatcher to return list of the best \"good enough\" matches.\n\n    word is a sequence for which close matches are desired (typically a\n    string).\n\n    possibilities is a list of sequences against which to match word\n    (typically a list of strings).\n\n    Optional arg n (default 3) is the maximum number of close matches to\n    return.  n must be > 0.\n\n    Optional arg cutoff (default 0.6) is a float in [0, 1].  Possibilities\n    that don't score at least that similar to word are ignored.\n\n    The best (no more than n) matches among the possibilities are returned\n    in a list, sorted by similarity score, most similar first.\n\n    >>> get_close_matches(\"appel\", [\"ape\", \"apple\", \"peach\", \"puppy\"])\n    ['apple', 'ape']\n    >>> import keyword as _keyword\n    >>> get_close_matches(\"wheel\", _keyword.kwlist)\n    ['while']\n    >>> get_close_matches(\"apple\", _keyword.kwlist)\n    []\n    >>> get_close_matches(\"accept\", _keyword.kwlist)\n    ['except']\n    \"\"\"\n\n    if not n >  0:\n        raise ValueError(\"n must be > 0: %r\" % (n,))\n    if not 0.0 <= cutoff <= 1.0:\n        raise ValueError(\"cutoff must be in [0.0, 1.0]: %r\" % (cutoff,))\n    result = []\n    s = SequenceMatcher()\n    s.set_seq2(word)\n    for x in possibilities:\n        s.set_seq1(x)\n        if s.real_quick_ratio() >= cutoff and \\\n           s.quick_ratio() >= cutoff and \\\n           s.ratio() >= cutoff:\n            result.append((s.ratio(), x))\n\n    # Move the best scorers to head of list\n    result = heapq.nlargest(n, result)\n    # Strip scores for the best n matches\n    return [x for score, x in result]"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _count_leading(line, ch):\n\n    i, n = 0, len(line)\n    while i < n and line[i] == ch:\n        i += 1\n    return i", "response": "Return number of ch characters at the start of line."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _format_range_unified(start, stop):\n    'Convert range to the \"ed\" format'\n    # Per the diff spec at http://www.unix.org/single_unix_specification/\n    beginning = start + 1     # lines start numbering with one\n    length = stop - start\n    if length == 1:\n        # return '{}'.format(beginning)\n        return '%s' % (beginning)\n    if not length:\n        beginning -= 1        # empty ranges begin at line just before the range\n    return '%s,%s' % (beginning, length)", "response": "Convert range to the ed format"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef unified_diff(a, b, fromfile='', tofile='', fromfiledate='',\n                 tofiledate='', n=3, lineterm='\\n'):\n    r\"\"\"\n    Compare two sequences of lines; generate the delta as a unified diff.\n\n    Unified diffs are a compact way of showing line changes and a few\n    lines of context.  The number of context lines is set by 'n' which\n    defaults to three.\n\n    By default, the diff control lines (those with ---, +++, or @@) are\n    created with a trailing newline.  This is helpful so that inputs\n    created from file.readlines() result in diffs that are suitable for\n    file.writelines() since both the inputs and outputs have trailing\n    newlines.\n\n    For inputs that do not have trailing newlines, set the lineterm\n    argument to \"\" so that the output will be uniformly newline free.\n\n    The unidiff format normally has a header for filenames and modification\n    times.  Any or all of these may be specified using strings for\n    'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'.\n    The modification times are normally expressed in the ISO 8601 format.\n\n    Example:\n\n    >>> for line in unified_diff('one two three four'.split(),\n    ...             'zero one tree four'.split(), 'Original', 'Current',\n    ...             '2005-01-26 23:30:50', '2010-04-02 10:20:52',\n    ...             lineterm=''):\n    ...     print line                  # doctest: +NORMALIZE_WHITESPACE\n    --- Original        2005-01-26 23:30:50\n    +++ Current         2010-04-02 10:20:52\n    @@ -1,4 +1,4 @@\n    +zero\n     one\n    -two\n    -three\n    +tree\n     four\n    \"\"\"\n\n    started = False\n    for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n):\n        if not started:\n            started = True\n            # fromdate = '\\t{}'.format(fromfiledate) if fromfiledate else ''\n            fromdate = '\\t%s' % (fromfiledate) if fromfiledate else ''\n            # todate = '\\t{}'.format(tofiledate) if tofiledate else ''\n            todate = '\\t%s' % (tofiledate) if tofiledate else ''\n            # yield '--- {}{}{}'.format(fromfile, fromdate, lineterm)\n            yield '--- %s%s%s' % (fromfile, fromdate, lineterm)\n            # yield '+++ {}{}{}'.format(tofile, todate, lineterm)\n            yield '+++ %s%s%s' % (tofile, todate, lineterm)\n\n        first, last = group[0], group[-1]\n        file1_range = _format_range_unified(first[1], last[2])\n        file2_range = _format_range_unified(first[3], last[4])\n        # yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm)\n        yield '@@ -%s +%s @@%s' % (file1_range, file2_range, lineterm)\n\n        for tag, i1, i2, j1, j2 in group:\n            if tag == 'equal':\n                for line in a[i1:i2]:\n                    yield ' ' + line\n                continue\n            if tag in ('replace', 'delete'):\n                for line in a[i1:i2]:\n                    yield '-' + line\n            if tag in ('replace', 'insert'):\n                for line in b[j1:j2]:\n                    yield '+' + line", "response": "r Generate a unified diff between two sequences of lines."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _format_range_context(start, stop):\n    'Convert range to the \"ed\" format'\n    # Per the diff spec at http://www.unix.org/single_unix_specification/\n    beginning = start + 1     # lines start numbering with one\n    length = stop - start\n    if not length:\n        beginning -= 1        # empty ranges begin at line just before the range\n    if length <= 1:\n        # return '{}'.format(beginning)\n        return '%s' % (beginning)\n    # return '{},{}'.format(beginning, beginning + length - 1)\n    return '%s,%s' % (beginning, beginning + length - 1)", "response": "Convert range to the ed format"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef context_diff(a, b, fromfile='', tofile='',\n                 fromfiledate='', tofiledate='', n=3, lineterm='\\n'):\n    r\"\"\"\n    Compare two sequences of lines; generate the delta as a context diff.\n\n    Context diffs are a compact way of showing line changes and a few\n    lines of context.  The number of context lines is set by 'n' which\n    defaults to three.\n\n    By default, the diff control lines (those with *** or ---) are\n    created with a trailing newline.  This is helpful so that inputs\n    created from file.readlines() result in diffs that are suitable for\n    file.writelines() since both the inputs and outputs have trailing\n    newlines.\n\n    For inputs that do not have trailing newlines, set the lineterm\n    argument to \"\" so that the output will be uniformly newline free.\n\n    The context diff format normally has a header for filenames and\n    modification times.  Any or all of these may be specified using\n    strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'.\n    The modification times are normally expressed in the ISO 8601 format.\n    If not specified, the strings default to blanks.\n\n    Example:\n\n    >>> print ''.join(context_diff('one\\ntwo\\nthree\\nfour\\n'.splitlines(1),\n    ...       'zero\\none\\ntree\\nfour\\n'.splitlines(1), 'Original', 'Current')),\n    *** Original\n    --- Current\n    ***************\n    *** 1,4 ****\n      one\n    ! two\n    ! three\n      four\n    --- 1,4 ----\n    + zero\n      one\n    ! tree\n      four\n    \"\"\"\n\n    prefix = dict(insert='+ ', delete='- ', replace='! ', equal='  ')\n    started = False\n    for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n):\n        if not started:\n            started = True\n            # fromdate = '\\t{}'.format(fromfiledate) if fromfiledate else ''\n            fromdate = '\\t%s' % (fromfiledate) if fromfiledate else ''\n            # todate = '\\t{}'.format(tofiledate) if tofiledate else ''\n            todate = '\\t%s' % (tofiledate) if tofiledate else ''\n            # yield '*** {}{}{}'.format(fromfile, fromdate, lineterm)\n            yield '*** %s%s%s' % (fromfile, fromdate, lineterm)\n            # yield '--- {}{}{}'.format(tofile, todate, lineterm)\n            yield '--- %s%s%s' % (tofile, todate, lineterm)\n\n        first, last = group[0], group[-1]\n        yield '***************' + lineterm\n\n        file1_range = _format_range_context(first[1], last[2])\n        # yield '*** {} ****{}'.format(file1_range, lineterm)\n        yield '*** %s ****%s' % (file1_range, lineterm)\n\n        if any(tag in ('replace', 'delete') for tag, _, _, _, _ in group):\n            for tag, i1, i2, _, _ in group:\n                if tag != 'insert':\n                    for line in a[i1:i2]:\n                        yield prefix[tag] + line\n\n        file2_range = _format_range_context(first[3], last[4])\n        # yield '--- {} ----{}'.format(file2_range, lineterm)\n        yield '--- %s ----%s' % (file2_range, lineterm)\n\n        if any(tag in ('replace', 'insert') for tag, _, _, _, _ in group):\n            for tag, _, _, j1, j2 in group:\n                if tag != 'delete':\n                    for line in b[j1:j2]:\n                        yield prefix[tag] + line", "response": "r Generate a context diff between two sequences of lines."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK):\n    return Differ(linejunk, charjunk).compare(a, b)", "response": "r Compare two lists of strings and return a Differ - style delta."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _mdiff(fromlines, tolines, context=None, linejunk=None,\n           charjunk=IS_CHARACTER_JUNK):\n    r\"\"\"Returns generator yielding marked up from/to side by side differences.\n\n    Arguments:\n    fromlines -- list of text lines to compared to tolines\n    tolines -- list of text lines to be compared to fromlines\n    context -- number of context lines to display on each side of difference,\n               if None, all from/to text lines will be generated.\n    linejunk -- passed on to ndiff (see ndiff documentation)\n    charjunk -- passed on to ndiff (see ndiff documentation)\n\n    This function returns an iterator which returns a tuple:\n    (from line tuple, to line tuple, boolean flag)\n\n    from/to line tuple -- (line num, line text)\n        line num -- integer or None (to indicate a context separation)\n        line text -- original line text with following markers inserted:\n            '\\0+' -- marks start of added text\n            '\\0-' -- marks start of deleted text\n            '\\0^' -- marks start of changed text\n            '\\1' -- marks end of added/deleted/changed text\n\n    boolean flag -- None indicates context separation, True indicates\n        either \"from\" or \"to\" line contains a change, otherwise False.\n\n    This function/iterator was originally developed to generate side by side\n    file difference for making HTML pages (see HtmlDiff class for example\n    usage).\n\n    Note, this function utilizes the ndiff function to generate the side by\n    side difference markup.  Optional ndiff arguments may be passed to this\n    function and they in turn will be passed to ndiff.\n    \"\"\"\n    import re\n\n    # regular expression for finding intraline change indices\n    change_re = re.compile('(\\++|\\-+|\\^+)')\n\n    # create the difference iterator to generate the differences\n    diff_lines_iterator = ndiff(fromlines,tolines,linejunk,charjunk)\n\n    def _make_line(lines, format_key, side, num_lines=[0,0]):\n        \"\"\"Returns line of text with user's change markup and line formatting.\n\n        lines -- list of lines from the ndiff generator to produce a line of\n                 text from.  When producing the line of text to return, the\n                 lines used are removed from this list.\n        format_key -- '+' return first line in list with \"add\" markup around\n                          the entire line.\n                      '-' return first line in list with \"delete\" markup around\n                          the entire line.\n                      '?' return first line in list with add/delete/change\n                          intraline markup (indices obtained from second line)\n                      None return first line in list with no markup\n        side -- indice into the num_lines list (0=from,1=to)\n        num_lines -- from/to current line number.  This is NOT intended to be a\n                     passed parameter.  It is present as a keyword argument to\n                     maintain memory of the current line numbers between calls\n                     of this function.\n\n        Note, this function is purposefully not defined at the module scope so\n        that data it needs from its parent function (within whose context it\n        is defined) does not need to be of module scope.\n        \"\"\"\n        num_lines[side] += 1\n        # Handle case where no user markup is to be added, just return line of\n        # text with user's line format to allow for usage of the line number.\n        if format_key is None:\n            return (num_lines[side],lines.pop(0)[2:])\n        # Handle case of intraline changes\n        if format_key == '?':\n            text, markers = lines.pop(0), lines.pop(0)\n            # find intraline changes (store change type and indices in tuples)\n            sub_info = []\n            def record_sub_info(match_object,sub_info=sub_info):\n                sub_info.append([match_object.group(1)[0],match_object.span()])\n                return match_object.group(1)\n            change_re.sub(record_sub_info,markers)\n            # process each tuple inserting our special marks that won't be\n            # noticed by an xml/html escaper.\n            for key,(begin,end) in sub_info[::-1]:\n                text = text[0:begin]+'\\0'+key+text[begin:end]+'\\1'+text[end:]\n            text = text[2:]\n        # Handle case of add/delete entire line\n        else:\n            text = lines.pop(0)[2:]\n            # if line of text is just a newline, insert a space so there is\n            # something for the user to highlight and see.\n            if not text:\n                text = ' '\n            # insert marks that won't be noticed by an xml/html escaper.\n            text = '\\0' + format_key + text + '\\1'\n        # Return line of text, first allow user's line formatter to do its\n        # thing (such as adding the line number) then replace the special\n        # marks with what the user's change markup.\n        return (num_lines[side],text)\n\n    def _line_iterator():\n        \"\"\"Yields from/to lines of text with a change indication.\n\n        This function is an iterator.  It itself pulls lines from a\n        differencing iterator, processes them and yields them.  When it can\n        it yields both a \"from\" and a \"to\" line, otherwise it will yield one\n        or the other.  In addition to yielding the lines of from/to text, a\n        boolean flag is yielded to indicate if the text line(s) have\n        differences in them.\n\n        Note, this function is purposefully not defined at the module scope so\n        that data it needs from its parent function (within whose context it\n        is defined) does not need to be of module scope.\n        \"\"\"\n        lines = []\n        num_blanks_pending, num_blanks_to_yield = 0, 0\n        while True:\n            # Load up next 4 lines so we can look ahead, create strings which\n            # are a concatenation of the first character of each of the 4 lines\n            # so we can do some very readable comparisons.\n            while len(lines) < 4:\n                try:\n                    lines.append(diff_lines_iterator.next())\n                except StopIteration:\n                    lines.append('X')\n            s = ''.join([line[0] for line in lines])\n            if s.startswith('X'):\n                # When no more lines, pump out any remaining blank lines so the\n                # corresponding add/delete lines get a matching blank line so\n                # all line pairs get yielded at the next level.\n                num_blanks_to_yield = num_blanks_pending\n            elif s.startswith('-?+?'):\n                # simple intraline change\n                yield _make_line(lines,'?',0), _make_line(lines,'?',1), True\n                continue\n            elif s.startswith('--++'):\n                # in delete block, add block coming: we do NOT want to get\n                # caught up on blank lines yet, just process the delete line\n                num_blanks_pending -= 1\n                yield _make_line(lines,'-',0), None, True\n                continue\n            elif s.startswith(('--?+', '--+', '- ')):\n                # in delete block and see an intraline change or unchanged line\n                # coming: yield the delete line and then blanks\n                from_line,to_line = _make_line(lines,'-',0), None\n                num_blanks_to_yield,num_blanks_pending = num_blanks_pending-1,0\n            elif s.startswith('-+?'):\n                # intraline change\n                yield _make_line(lines,None,0), _make_line(lines,'?',1), True\n                continue\n            elif s.startswith('-?+'):\n                # intraline change\n                yield _make_line(lines,'?',0), _make_line(lines,None,1), True\n                continue\n            elif s.startswith('-'):\n                # delete FROM line\n                num_blanks_pending -= 1\n                yield _make_line(lines,'-',0), None, True\n                continue\n            elif s.startswith('+--'):\n                # in add block, delete block coming: we do NOT want to get\n                # caught up on blank lines yet, just process the add line\n                num_blanks_pending += 1\n                yield None, _make_line(lines,'+',1), True\n                continue\n            elif s.startswith(('+ ', '+-')):\n                # will be leaving an add block: yield blanks then add line\n                from_line, to_line = None, _make_line(lines,'+',1)\n                num_blanks_to_yield,num_blanks_pending = num_blanks_pending+1,0\n            elif s.startswith('+'):\n                # inside an add block, yield the add line\n                num_blanks_pending += 1\n                yield None, _make_line(lines,'+',1), True\n                continue\n            elif s.startswith(' '):\n                # unchanged text, yield it to both sides\n                yield _make_line(lines[:],None,0),_make_line(lines,None,1),False\n                continue\n            # Catch up on the blank lines so when we yield the next from/to\n            # pair, they are lined up.\n            while(num_blanks_to_yield < 0):\n                num_blanks_to_yield += 1\n                yield None,('','\\n'),True\n            while(num_blanks_to_yield > 0):\n                num_blanks_to_yield -= 1\n                yield ('','\\n'),None,True\n            if s.startswith('X'):\n                raise StopIteration\n            else:\n                yield from_line,to_line,True\n\n    def _line_pair_iterator():\n        \"\"\"Yields from/to lines of text with a change indication.\n\n        This function is an iterator.  It itself pulls lines from the line\n        iterator.  Its difference from that iterator is that this function\n        always yields a pair of from/to text lines (with the change\n        indication).  If necessary it will collect single from/to lines\n        until it has a matching pair from/to pair to yield.\n\n        Note, this function is purposefully not defined at the module scope so\n        that data it needs from its parent function (within whose context it\n        is defined) does not need to be of module scope.\n        \"\"\"\n        line_iterator = _line_iterator()\n        fromlines,tolines=[],[]\n        while True:\n            # Collecting lines of text until we have a from/to pair\n            while (len(fromlines)==0 or len(tolines)==0):\n                from_line, to_line, found_diff =line_iterator.next()\n                if from_line is not None:\n                    fromlines.append((from_line,found_diff))\n                if to_line is not None:\n                    tolines.append((to_line,found_diff))\n            # Once we have a pair, remove them from the collection and yield it\n            from_line, fromDiff = fromlines.pop(0)\n            to_line, to_diff = tolines.pop(0)\n            yield (from_line,to_line,fromDiff or to_diff)\n\n    # Handle case where user does not want context differencing, just yield\n    # them up without doing anything else with them.\n    line_pair_iterator = _line_pair_iterator()\n    if context is None:\n        while True:\n            yield line_pair_iterator.next()\n    # Handle case where user wants context differencing.  We must do some\n    # storage of lines until we know for sure that they are to be yielded.\n    else:\n        context += 1\n        lines_to_write = 0\n        while True:\n            # Store lines up until we find a difference, note use of a\n            # circular queue because we only need to keep around what\n            # we need for context.\n            index, contextLines = 0, [None]*(context)\n            found_diff = False\n            while(found_diff is False):\n                from_line, to_line, found_diff = line_pair_iterator.next()\n                i = index % context\n                contextLines[i] = (from_line, to_line, found_diff)\n                index += 1\n            # Yield lines that we have collected so far, but first yield\n            # the user's separator.\n            if index > context:\n                yield None, None, None\n                lines_to_write = context\n            else:\n                lines_to_write = index\n                index = 0\n            while(lines_to_write):\n                i = index % context\n                index += 1\n                yield contextLines[i]\n                lines_to_write -= 1\n            # Now yield the context lines after the change\n            lines_to_write = context-1\n            while(lines_to_write):\n                from_line, to_line, found_diff = line_pair_iterator.next()\n                # If another change within the context, extend the context\n                if found_diff:\n                    lines_to_write = context-1\n                else:\n                    lines_to_write -= 1\n                yield from_line, to_line, found_diff", "response": "r Returns an iterator that generates the diff between two lists of lines."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nmake a new Match object from a sequence or iterable", "response": "def _make(cls, iterable, new=tuple.__new__, len=len):\n        'Make a new Match object from a sequence or iterable'\n        result = new(cls, iterable)\n        if len(result) != 3:\n            raise TypeError('Expected 3 arguments, got %d' % len(result))\n        return result"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_seq1(self, a):\n\n        if a is self.a:\n            return\n        self.a = a\n        self.matching_blocks = self.opcodes = None", "response": "Set the first sequence to be compared."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsets the second sequence to be compared.", "response": "def set_seq2(self, b):\n        \"\"\"Set the second sequence to be compared.\n\n        The first sequence to be compared is not changed.\n\n        >>> s = SequenceMatcher(None, \"abcd\", \"bcde\")\n        >>> s.ratio()\n        0.75\n        >>> s.set_seq2(\"abcd\")\n        >>> s.ratio()\n        1.0\n        >>>\n\n        SequenceMatcher computes and caches detailed information about the\n        second sequence, so if you want to compare one sequence S against\n        many sequences, use .set_seq2(S) once and call .set_seq1(x)\n        repeatedly for each of the other sequences.\n\n        See also set_seqs() and set_seq1().\n        \"\"\"\n\n        if b is self.b:\n            return\n        self.b = b\n        self.matching_blocks = self.opcodes = None\n        self.fullbcount = None\n        self.__chain_b()"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef find_longest_match(self, alo, ahi, blo, bhi):\n\n        # CAUTION:  stripping common prefix or suffix would be incorrect.\n        # E.g.,\n        #    ab\n        #    acab\n        # Longest matching block is \"ab\", but if common prefix is\n        # stripped, it's \"a\" (tied with \"b\").  UNIX(tm) diff does so\n        # strip, so ends up claiming that ab is changed to acab by\n        # inserting \"ca\" in the middle.  That's minimal but unintuitive:\n        # \"it's obvious\" that someone inserted \"ac\" at the front.\n        # Windiff ends up at the same place as diff, but by pairing up\n        # the unique 'b's and then matching the first two 'a's.\n\n        a, b, b2j, isbjunk = self.a, self.b, self.b2j, self.isbjunk\n        besti, bestj, bestsize = alo, blo, 0\n        # find longest junk-free match\n        # during an iteration of the loop, j2len[j] = length of longest\n        # junk-free match ending with a[i-1] and b[j]\n        j2len = {}\n        nothing = []\n        for i in xrange(alo, ahi):\n            # look at all instances of a[i] in b; note that because\n            # b2j has no junk keys, the loop is skipped if a[i] is junk\n            j2lenget = j2len.get\n            newj2len = {}\n            for j in b2j.get(a[i], nothing):\n                # a[i] matches b[j]\n                if j < blo:\n                    continue\n                if j >= bhi:\n                    break\n                k = newj2len[j] = j2lenget(j-1, 0) + 1\n                if k > bestsize:\n                    besti, bestj, bestsize = i-k+1, j-k+1, k\n            j2len = newj2len\n\n        # Extend the best by non-junk elements on each end.  In particular,\n        # \"popular\" non-junk elements aren't in b2j, which greatly speeds\n        # the inner loop above, but also means \"the best\" match so far\n        # doesn't contain any junk *or* popular non-junk elements.\n        while besti > alo and bestj > blo and \\\n              not isbjunk(b[bestj-1]) and \\\n              a[besti-1] == b[bestj-1]:\n            besti, bestj, bestsize = besti-1, bestj-1, bestsize+1\n        while besti+bestsize < ahi and bestj+bestsize < bhi and \\\n              not isbjunk(b[bestj+bestsize]) and \\\n              a[besti+bestsize] == b[bestj+bestsize]:\n            bestsize += 1\n\n        # Now that we have a wholly interesting match (albeit possibly\n        # empty!), we may as well suck up the matching junk on each\n        # side of it too.  Can't think of a good reason not to, and it\n        # saves post-processing the (possibly considerable) expense of\n        # figuring out what to do with it.  In the case of an empty\n        # interesting match, this is clearly the right thing to do,\n        # because no other kind of match is possible in the regions.\n        while besti > alo and bestj > blo and \\\n              isbjunk(b[bestj-1]) and \\\n              a[besti-1] == b[bestj-1]:\n            besti, bestj, bestsize = besti-1, bestj-1, bestsize+1\n        while besti+bestsize < ahi and bestj+bestsize < bhi and \\\n              isbjunk(b[bestj+bestsize]) and \\\n              a[besti+bestsize] == b[bestj+bestsize]:\n            bestsize = bestsize + 1\n\n        return Match(besti, bestj, bestsize)", "response": "Find the longest matching block in a and b."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a list of triples describing matching subsequences.", "response": "def get_matching_blocks(self):\n        \"\"\"Return list of triples describing matching subsequences.\n\n        Each triple is of the form (i, j, n), and means that\n        a[i:i+n] == b[j:j+n].  The triples are monotonically increasing in\n        i and in j.  New in Python 2.5, it's also guaranteed that if\n        (i, j, n) and (i', j', n') are adjacent triples in the list, and\n        the second is not the last triple in the list, then i+n != i' or\n        j+n != j'.  IOW, adjacent triples never describe adjacent equal\n        blocks.\n\n        The last triple is a dummy, (len(a), len(b), 0), and is the only\n        triple with n==0.\n\n        >>> s = SequenceMatcher(None, \"abxcd\", \"abcd\")\n        >>> s.get_matching_blocks()\n        [Match(a=0, b=0, size=2), Match(a=3, b=2, size=2), Match(a=5, b=4, size=0)]\n        \"\"\"\n\n        if self.matching_blocks is not None:\n            return self.matching_blocks\n        la, lb = len(self.a), len(self.b)\n\n        # This is most naturally expressed as a recursive algorithm, but\n        # at least one user bumped into extreme use cases that exceeded\n        # the recursion limit on their box.  So, now we maintain a list\n        # ('queue`) of blocks we still need to look at, and append partial\n        # results to `matching_blocks` in a loop; the matches are sorted\n        # at the end.\n        queue = [(0, la, 0, lb)]\n        matching_blocks = []\n        while queue:\n            alo, ahi, blo, bhi = queue.pop()\n            i, j, k = x = self.find_longest_match(alo, ahi, blo, bhi)\n            # a[alo:i] vs b[blo:j] unknown\n            # a[i:i+k] same as b[j:j+k]\n            # a[i+k:ahi] vs b[j+k:bhi] unknown\n            if k:   # if k is 0, there was no matching block\n                matching_blocks.append(x)\n                if alo < i and blo < j:\n                    queue.append((alo, i, blo, j))\n                if i+k < ahi and j+k < bhi:\n                    queue.append((i+k, ahi, j+k, bhi))\n        matching_blocks.sort()\n\n        # It's possible that we have adjacent equal blocks in the\n        # matching_blocks list now.  Starting with 2.5, this code was added\n        # to collapse them.\n        i1 = j1 = k1 = 0\n        non_adjacent = []\n        for i2, j2, k2 in matching_blocks:\n            # Is this block adjacent to i1, j1, k1?\n            if i1 + k1 == i2 and j1 + k1 == j2:\n                # Yes, so collapse them -- this just increases the length of\n                # the first block by the length of the second, and the first\n                # block so lengthened remains the block to compare against.\n                k1 += k2\n            else:\n                # Not adjacent.  Remember the first block (k1==0 means it's\n                # the dummy we started with), and make the second block the\n                # new block to compare against.\n                if k1:\n                    non_adjacent.append((i1, j1, k1))\n                i1, j1, k1 = i2, j2, k2\n        if k1:\n            non_adjacent.append((i1, j1, k1))\n\n        non_adjacent.append( (la, lb, 0) )\n        self.matching_blocks = map(Match._make, non_adjacent)\n        return self.matching_blocks"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_opcodes(self):\n\n        if self.opcodes is not None:\n            return self.opcodes\n        i = j = 0\n        self.opcodes = answer = []\n        for ai, bj, size in self.get_matching_blocks():\n            # invariant:  we've pumped out correct diffs to change\n            # a[:i] into b[:j], and the next matching block is\n            # a[ai:ai+size] == b[bj:bj+size].  So we need to pump\n            # out a diff to change a[i:ai] into b[j:bj], pump out\n            # the matching block, and move (i,j) beyond the match\n            tag = ''\n            if i < ai and j < bj:\n                tag = 'replace'\n            elif i < ai:\n                tag = 'delete'\n            elif j < bj:\n                tag = 'insert'\n            if tag:\n                answer.append( (tag, i, ai, j, bj) )\n            i, j = ai+size, bj+size\n            # the list of matching blocks is terminated by a\n            # sentinel with size 0\n            if size:\n                answer.append( ('equal', ai, i, bj, j) )\n        return answer", "response": "Return a list of 5 - tuples describing how to turn a into b."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_grouped_opcodes(self, n=3):\n\n        codes = self.get_opcodes()\n        if not codes:\n            codes = [(\"equal\", 0, 1, 0, 1)]\n        # Fixup leading and trailing groups if they show no changes.\n        if codes[0][0] == 'equal':\n            tag, i1, i2, j1, j2 = codes[0]\n            codes[0] = tag, max(i1, i2-n), i2, max(j1, j2-n), j2\n        if codes[-1][0] == 'equal':\n            tag, i1, i2, j1, j2 = codes[-1]\n            codes[-1] = tag, i1, min(i2, i1+n), j1, min(j2, j1+n)\n\n        nn = n + n\n        group = []\n        for tag, i1, i2, j1, j2 in codes:\n            # End the current group and start a new one whenever\n            # there is a large range with no changes.\n            if tag == 'equal' and i2-i1 > nn:\n                group.append((tag, i1, min(i2, i1+n), j1, min(j2, j1+n)))\n                yield group\n                group = []\n                i1, j1 = max(i1, i2-n), max(j1, j2-n)\n            group.append((tag, i1, i2, j1 ,j2))\n        if group and not (len(group)==1 and group[0][0] == 'equal'):\n            yield group", "response": "Return a generator of grouped change clusters."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a measure of the sequences s similarity.", "response": "def ratio(self):\n        \"\"\"Return a measure of the sequences' similarity (float in [0,1]).\n\n        Where T is the total number of elements in both sequences, and\n        M is the number of matches, this is 2.0*M / T.\n        Note that this is 1 if the sequences are identical, and 0 if\n        they have nothing in common.\n\n        .ratio() is expensive to compute if you haven't already computed\n        .get_matching_blocks() or .get_opcodes(), in which case you may\n        want to try .quick_ratio() or .real_quick_ratio() first to get an\n        upper bound.\n\n        >>> s = SequenceMatcher(None, \"abcd\", \"bcde\")\n        >>> s.ratio()\n        0.75\n        >>> s.quick_ratio()\n        0.75\n        >>> s.real_quick_ratio()\n        1.0\n        \"\"\"\n\n        matches = reduce(lambda sum, triple: sum + triple[-1],\n                         self.get_matching_blocks(), 0)\n        return _calculate_ratio(matches, len(self.a) + len(self.b))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef quick_ratio(self):\n\n        # viewing a and b as multisets, set matches to the cardinality\n        # of their intersection; this counts the number of matches\n        # without regard to order, so is clearly an upper bound\n        if self.fullbcount is None:\n            self.fullbcount = fullbcount = {}\n            for elt in self.b:\n                fullbcount[elt] = fullbcount.get(elt, 0) + 1\n        fullbcount = self.fullbcount\n        # avail[x] is the number of times x appears in 'b' less the\n        # number of times we've seen it in 'a' so far ... kinda\n        avail = {}\n        availhas, matches = avail.__contains__, 0\n        for elt in self.a:\n            if availhas(elt):\n                numb = avail[elt]\n            else:\n                numb = fullbcount.get(elt, 0)\n            avail[elt] = numb - 1\n            if numb > 0:\n                matches = matches + 1\n        return _calculate_ratio(matches, len(self.a) + len(self.b))", "response": "Return an upper bound on ratio() relatively quickly."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef real_quick_ratio(self):\n\n        la, lb = len(self.a), len(self.b)\n        # can't have more matches than the number of elements in the\n        # shorter sequence\n        return _calculate_ratio(min(la, lb), la + lb)", "response": "Return an upper bound on ratio() very quickly."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _dump(self, tag, x, lo, hi):\n        for i in xrange(lo, hi):\n            yield '%s %s' % (tag, x[i])", "response": "Generate comparison results for a same - tagged range."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef make_file(self,fromlines,tolines,fromdesc='',todesc='',context=False,\n                  numlines=5):\n        \"\"\"Returns HTML file of side by side comparison with change highlights\n\n        Arguments:\n        fromlines -- list of \"from\" lines\n        tolines -- list of \"to\" lines\n        fromdesc -- \"from\" file column header string\n        todesc -- \"to\" file column header string\n        context -- set to True for contextual differences (defaults to False\n            which shows full differences).\n        numlines -- number of context lines.  When context is set True,\n            controls number of lines displayed before and after the change.\n            When context is False, controls the number of lines to place\n            the \"next\" link anchors before the next change (so click of\n            \"next\" link jumps to just before the change).\n        \"\"\"\n\n        return self._file_template % dict(\n            styles = self._styles,\n            legend = self._legend,\n            table = self.make_table(fromlines,tolines,fromdesc,todesc,\n                                    context=context,numlines=numlines))", "response": "Returns HTML file of side by side comparison with change highlights"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn fromlines and tolines with tabs expanded and newlines removed.", "response": "def _tab_newline_replace(self,fromlines,tolines):\n        \"\"\"Returns from/to line lists with tabs expanded and newlines removed.\n\n        Instead of tab characters being replaced by the number of spaces\n        needed to fill in to the next tab stop, this function will fill\n        the space with tab characters.  This is done so that the difference\n        algorithms can identify changes in a file when tabs are replaced by\n        spaces and vice versa.  At the end of the HTML generation, the tab\n        characters will be replaced with a nonbreakable space.\n        \"\"\"\n        def expand_tabs(line):\n            # hide real spaces\n            line = line.replace(' ','\\0')\n            # expand tabs into spaces\n            line = line.expandtabs(self._tabsize)\n            # replace spaces from expanded tabs back into tab characters\n            # (we'll replace them with markup after we do differencing)\n            line = line.replace(' ','\\t')\n            return line.replace('\\0',' ').rstrip('\\n')\n        fromlines = [expand_tabs(line) for line in fromlines]\n        tolines = [expand_tabs(line) for line in tolines]\n        return fromlines,tolines"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _split_line(self,data_list,line_num,text):\n        # if blank line or context separator, just add it to the output list\n        if not line_num:\n            data_list.append((line_num,text))\n            return\n\n        # if line text doesn't need wrapping, just add it to the output list\n        size = len(text)\n        max = self._wrapcolumn\n        if (size <= max) or ((size -(text.count('\\0')*3)) <= max):\n            data_list.append((line_num,text))\n            return\n\n        # scan text looking for the wrap point, keeping track if the wrap\n        # point is inside markers\n        i = 0\n        n = 0\n        mark = ''\n        while n < max and i < size:\n            if text[i] == '\\0':\n                i += 1\n                mark = text[i]\n                i += 1\n            elif text[i] == '\\1':\n                i += 1\n                mark = ''\n            else:\n                i += 1\n                n += 1\n\n        # wrap point is inside text, break it up into separate lines\n        line1 = text[:i]\n        line2 = text[i:]\n\n        # if wrap point is inside markers, place end marker at end of first\n        # line and start marker at beginning of second line because each\n        # line will have its own table tag markup around it.\n        if mark:\n            line1 = line1 + '\\1'\n            line2 = '\\0' + mark + line2\n\n        # tack on first line onto the output list\n        data_list.append((line_num,line1))\n\n        # use this routine again to wrap the remaining text\n        self._split_line(data_list,'>',line2)", "response": "This function splits the input text into separate lines at the specified line number."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning an iterator that splits the mdiff text lines into fromdata todata and flag.", "response": "def _line_wrapper(self,diffs):\n        \"\"\"Returns iterator that splits (wraps) mdiff text lines\"\"\"\n\n        # pull from/to data and flags from mdiff iterator\n        for fromdata,todata,flag in diffs:\n            # check for context separators and pass them through\n            if flag is None:\n                yield fromdata,todata,flag\n                continue\n            (fromline,fromtext),(toline,totext) = fromdata,todata\n            # for each from/to line split it at the wrap column to form\n            # list of text lines.\n            fromlist,tolist = [],[]\n            self._split_line(fromlist,fromline,fromtext)\n            self._split_line(tolist,toline,totext)\n            # yield from/to line in pairs inserting blank lines as\n            # necessary when one side has more wrapped lines\n            while fromlist or tolist:\n                if fromlist:\n                    fromdata = fromlist.pop(0)\n                else:\n                    fromdata = ('',' ')\n                if tolist:\n                    todata = tolist.pop(0)\n                else:\n                    todata = ('',' ')\n                yield fromdata,todata,flag"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _collect_lines(self,diffs):\n\n        fromlist,tolist,flaglist = [],[],[]\n        # pull from/to data and flags from mdiff style iterator\n        for fromdata,todata,flag in diffs:\n            try:\n                # store HTML markup of the lines into the lists\n                fromlist.append(self._format_line(0,flag,*fromdata))\n                tolist.append(self._format_line(1,flag,*todata))\n            except TypeError:\n                # exceptions occur for lines where context separators go\n                fromlist.append(None)\n                tolist.append(None)\n            flaglist.append(flag)\n        return fromlist,tolist,flaglist", "response": "Collects the lines from the mdiff output into separate lists\nCOOKIENAME and\nCOOKIENAME into a list of lists\nCOOKIENAME and a list of lists\nCOOKIENAME."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating unique anchor prefixes for the current page.", "response": "def _make_prefix(self):\n        \"\"\"Create unique anchor prefixes\"\"\"\n\n        # Generate a unique anchor prefix so multiple tables\n        # can exist on the same HTML page without conflicts.\n        fromprefix = \"from%d_\" % HtmlDiff._default_prefix\n        toprefix = \"to%d_\" % HtmlDiff._default_prefix\n        HtmlDiff._default_prefix += 1\n        # store prefixes so line format method has access\n        self._prefix = [fromprefix,toprefix]"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconverts the flags in fromlist to tolist.", "response": "def _convert_flags(self,fromlist,tolist,flaglist,context,numlines):\n        \"\"\"Makes list of \"next\" links\"\"\"\n\n        # all anchor names will be generated using the unique \"to\" prefix\n        toprefix = self._prefix[1]\n\n        # process change flags, generating middle column of next anchors/links\n        next_id = ['']*len(flaglist)\n        next_href = ['']*len(flaglist)\n        num_chg, in_change = 0, False\n        last = 0\n        for i,flag in enumerate(flaglist):\n            if flag:\n                if not in_change:\n                    in_change = True\n                    last = i\n                    # at the beginning of a change, drop an anchor a few lines\n                    # (the context lines) before the change for the previous\n                    # link\n                    i = max([0,i-numlines])\n                    next_id[i] = ' id=\"difflib_chg_%s_%d\"' % (toprefix,num_chg)\n                    # at the beginning of a change, drop a link to the next\n                    # change\n                    num_chg += 1\n                    next_href[last] = '<a href=\"#difflib_chg_%s_%d\">n</a>' % (\n                         toprefix,num_chg)\n            else:\n                in_change = False\n        # check for cases where there is no content to avoid exceptions\n        if not flaglist:\n            flaglist = [False]\n            next_id = ['']\n            next_href = ['']\n            last = 0\n            if context:\n                fromlist = ['<td></td><td>&nbsp;No Differences Found&nbsp;</td>']\n                tolist = fromlist\n            else:\n                fromlist = tolist = ['<td></td><td>&nbsp;Empty File&nbsp;</td>']\n        # if not a change on first line, drop a link\n        if not flaglist[0]:\n            next_href[0] = '<a href=\"#difflib_chg_%s_0\">f</a>' % toprefix\n        # redo the last link to link to the top\n        next_href[last] = '<a href=\"#difflib_chg_%s_top\">t</a>' % (toprefix)\n\n        return fromlist,tolist,flaglist,next_href,next_id"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns an HTML table of side by side comparison with change highlights.", "response": "def make_table(self,fromlines,tolines,fromdesc='',todesc='',context=False,\n                   numlines=5):\n        \"\"\"Returns HTML table of side by side comparison with change highlights\n\n        Arguments:\n        fromlines -- list of \"from\" lines\n        tolines -- list of \"to\" lines\n        fromdesc -- \"from\" file column header string\n        todesc -- \"to\" file column header string\n        context -- set to True for contextual differences (defaults to False\n            which shows full differences).\n        numlines -- number of context lines.  When context is set True,\n            controls number of lines displayed before and after the change.\n            When context is False, controls the number of lines to place\n            the \"next\" link anchors before the next change (so click of\n            \"next\" link jumps to just before the change).\n        \"\"\"\n\n        # make unique anchor prefixes so that multiple tables may exist\n        # on the same page without conflict.\n        self._make_prefix()\n\n        # change tabs to spaces before it gets more difficult after we insert\n        # markup\n        fromlines,tolines = self._tab_newline_replace(fromlines,tolines)\n\n        # create diffs iterator which generates side by side from/to data\n        if context:\n            context_lines = numlines\n        else:\n            context_lines = None\n        diffs = _mdiff(fromlines,tolines,context_lines,linejunk=self._linejunk,\n                      charjunk=self._charjunk)\n\n        # set up iterator to wrap lines that exceed desired width\n        if self._wrapcolumn:\n            diffs = self._line_wrapper(diffs)\n\n        # collect up from/to lines and flags into lists (also format the lines)\n        fromlist,tolist,flaglist = self._collect_lines(diffs)\n\n        # process change flags, generating middle column of next anchors/links\n        fromlist,tolist,flaglist,next_href,next_id = self._convert_flags(\n            fromlist,tolist,flaglist,context,numlines)\n\n        s = []\n        fmt = '            <tr><td class=\"diff_next\"%s>%s</td>%s' + \\\n              '<td class=\"diff_next\">%s</td>%s</tr>\\n'\n        for i in range(len(flaglist)):\n            if flaglist[i] is None:\n                # mdiff yields None on separator lines skip the bogus ones\n                # generated for the first line\n                if i > 0:\n                    s.append('        </tbody>        \\n        <tbody>\\n')\n            else:\n                s.append( fmt % (next_id[i],next_href[i],fromlist[i],\n                                           next_href[i],tolist[i]))\n        if fromdesc or todesc:\n            header_row = '<thead><tr>%s%s%s%s</tr></thead>' % (\n                '<th class=\"diff_next\"><br /></th>',\n                '<th colspan=\"2\" class=\"diff_header\">%s</th>' % fromdesc,\n                '<th class=\"diff_next\"><br /></th>',\n                '<th colspan=\"2\" class=\"diff_header\">%s</th>' % todesc)\n        else:\n            header_row = ''\n\n        table = self._table_template % dict(\n            data_rows=''.join(s),\n            header_row=header_row,\n            prefix=self._prefix[1])\n\n        return table.replace('\\0+','<span class=\"diff_add\">'). \\\n                     replace('\\0-','<span class=\"diff_sub\">'). \\\n                     replace('\\0^','<span class=\"diff_chg\">'). \\\n                     replace('\\1','</span>'). \\\n                     replace('\\t','&nbsp;')"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate and return a benchmark that runs work_func p times in parallel.", "response": "def _MakeParallelBenchmark(p, work_func, *args):\n  \"\"\"Create and return a benchmark that runs work_func p times in parallel.\"\"\"\n  def Benchmark(b):  # pylint: disable=missing-docstring\n    e = threading.Event()\n    def Target():\n      e.wait()\n      for _ in xrange(b.N / p):\n        work_func(*args)\n    threads = []\n    for _ in xrange(p):\n      t = threading.Thread(target=Target)\n      t.start()\n      threads.append(t)\n    b.ResetTimer()\n    e.set()\n    for t in threads:\n      t.join()\n  return Benchmark"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning an inline function with the given body.", "response": "def visit_function_inline(self, node):\n    \"\"\"Returns an GeneratedExpr for a function with the given body.\"\"\"\n    # First pass collects the names of locals used in this function. Do this in\n    # a separate pass so that we know whether to resolve a name as a local or a\n    # global during the second pass.\n    func_visitor = block.FunctionBlockVisitor(node)\n    for child in node.body:\n      func_visitor.visit(child)\n    func_block = block.FunctionBlock(self.block, node.name, func_visitor.vars,\n                                     func_visitor.is_generator)\n    visitor = StatementVisitor(func_block, self.future_node)\n    # Indent so that the function body is aligned with the goto labels.\n    with visitor.writer.indent_block():\n      visitor._visit_each(node.body)  # pylint: disable=protected-access\n\n    result = self.block.alloc_temp()\n    with self.block.alloc_temp('[]\u03c0g.Param') as func_args:\n      args = node.args\n      argc = len(args.args)\n      self.writer.write('{} = make([]\u03c0g.Param, {})'.format(\n          func_args.expr, argc))\n      # The list of defaults only contains args for which a default value is\n      # specified so pad it with None to make it the same length as args.\n      defaults = [None] * (argc - len(args.defaults)) + args.defaults\n      for i, (a, d) in enumerate(zip(args.args, defaults)):\n        with self.visit_expr(d) if d else expr.nil_expr as default:\n          tmpl = '$args[$i] = \u03c0g.Param{Name: $name, Def: $default}'\n          self.writer.write_tmpl(tmpl, args=func_args.expr, i=i,\n                                 name=util.go_str(a.arg), default=default.expr)\n      flags = []\n      if args.vararg:\n        flags.append('\u03c0g.CodeFlagVarArg')\n      if args.kwarg:\n        flags.append('\u03c0g.CodeFlagKWArg')\n      # The function object gets written to a temporary writer because we need\n      # it as an expression that we subsequently bind to some variable.\n      self.writer.write_tmpl(\n          '$result = \u03c0g.NewFunction(\u03c0g.NewCode($name, $filename, $args, '\n          '$flags, func(\u03c0F *\u03c0g.Frame, \u03c0Args []*\u03c0g.Object) '\n          '(*\u03c0g.Object, *\u03c0g.BaseException) {',\n          result=result.name, name=util.go_str(node.name),\n          filename=util.go_str(self.block.root.filename), args=func_args.expr,\n          flags=' | '.join(flags) if flags else 0)\n      with self.writer.indent_block():\n        for var in func_block.vars.values():\n          if var.type != block.Var.TYPE_GLOBAL:\n            fmt = 'var {0} *\u03c0g.Object = {1}; _ = {0}'\n            self.writer.write(fmt.format(\n                util.adjust_local_name(var.name), var.init_expr))\n        self.writer.write_temp_decls(func_block)\n        self.writer.write('var \u03c0R *\u03c0g.Object; _ = \u03c0R')\n        self.writer.write('var \u03c0E *\u03c0g.BaseException; _ = \u03c0E')\n        if func_block.is_generator:\n          self.writer.write(\n              'return \u03c0g.NewGenerator(\u03c0F, func(\u03c0Sent *\u03c0g.Object) '\n              '(*\u03c0g.Object, *\u03c0g.BaseException) {')\n          with self.writer.indent_block():\n            self.writer.write_block(func_block, visitor.writer.getvalue())\n            self.writer.write('return nil, \u03c0E')\n          self.writer.write('}).ToObject(), nil')\n        else:\n          self.writer.write_block(func_block, visitor.writer.getvalue())\n          self.writer.write(textwrap.dedent(\"\"\"\\\n              if \u03c0E != nil {\n              \\t\u03c0R = nil\n              } else if \u03c0R == nil {\n              \\t\u03c0R = \u03c0g.None\n              }\n              return \u03c0R, \u03c0E\"\"\"))\n      self.writer.write('}), \u03c0F.Globals()).ToObject()')\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nimporting a module and binds it to a variable.", "response": "def _import_and_bind(self, imp):\n    \"\"\"Generates code that imports a module and binds it to a variable.\n\n    Args:\n      imp: Import object representing an import of the form \"import x.y.z\" or\n          \"from x.y import z\". Expects only a single binding.\n    \"\"\"\n    # Acquire handles to the Code objects in each Go package and call\n    # ImportModule to initialize all modules.\n    with self.block.alloc_temp() as mod, \\\n        self.block.alloc_temp('[]*\u03c0g.Object') as mod_slice:\n      self.writer.write_checked_call2(\n          mod_slice, '\u03c0g.ImportModule(\u03c0F, {})', util.go_str(imp.name))\n\n      # Bind the imported modules or members to variables in the current scope.\n      for binding in imp.bindings:\n        if binding.bind_type == imputil.Import.MODULE:\n          self.writer.write('{} = {}[{}]'.format(\n              mod.name, mod_slice.expr, binding.value))\n          self.block.bind_var(self.writer, binding.alias, mod.expr)\n        else:\n          self.writer.write('{} = {}[{}]'.format(\n              mod.name, mod_slice.expr, imp.name.count('.')))\n          # Binding a member of the imported module.\n          with self.block.alloc_temp() as member:\n            self.writer.write_checked_call2(\n                member, '\u03c0g.GetAttr(\u03c0F, {}, {}, nil)',\n                mod.expr, self.block.root.intern(binding.value))\n            self.block.bind_var(self.writer, binding.alias, member.expr)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _write_except_dispatcher(self, exc, tb, handlers):\n    handler_labels = []\n    for i, except_node in enumerate(handlers):\n      handler_labels.append(self.block.genlabel())\n      if except_node.type:\n        with self.visit_expr(except_node.type) as type_,\\\n            self.block.alloc_temp('bool') as is_inst:\n          self.writer.write_checked_call2(\n              is_inst, '\u03c0g.IsInstance(\u03c0F, {}.ToObject(), {})', exc, type_.expr)\n          self.writer.write_tmpl(textwrap.dedent(\"\"\"\\\n              if $is_inst {\n              \\tgoto Label$label\n              }\"\"\"), is_inst=is_inst.expr, label=handler_labels[-1])\n      else:\n        # This is a bare except. It should be the last handler.\n        if i != len(handlers) - 1:\n          msg = \"default 'except:' must be last\"\n          raise util.ParseError(except_node, msg)\n        self.writer.write('goto Label{}'.format(handler_labels[-1]))\n    if handlers[-1].type:\n      # There's no bare except, so the fallback is to re-raise.\n      self.writer.write(\n          '\u03c0E = \u03c0F.Raise({}.ToObject(), nil, {}.ToObject())'.format(exc, tb))\n      self.writer.write('continue')\n    return handler_labels", "response": "Writes a dispatcher for an except statement."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nlists directory contents using cache.", "response": "def listdir(path):\n    \"\"\"List directory contents, using cache.\"\"\"\n    try:\n        cached_mtime, list = cache[path]\n        del cache[path]\n    except KeyError:\n        cached_mtime, list = -1, []\n    mtime = os.stat(path).st_mtime\n    if mtime != cached_mtime:\n        list = os.listdir(path)\n        list.sort()\n    cache[path] = mtime, list\n    return list"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef annotate(head, list):\n    for i in range(len(list)):\n        if os.path.isdir(os.path.join(head, list[i])):\n            list[i] = list[i] + '/'", "response": "Add '/' suffixes to directories."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nformats a Python o into a pretty - printed representation.", "response": "def pformat(o, indent=1, width=80, depth=None):\n    \"\"\"Format a Python o into a pretty-printed representation.\"\"\"\n    return PrettyPrinter(indent=indent, width=width, depth=depth).pformat(o)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef format(self, o, context, maxlevels, level):\n        return _safe_repr(o, context, maxlevels, level)", "response": "Format o for a specific context returning a string containing the representation of o and a flag indicating whether the representation is readable or not."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef Eps(value=None, loc=None):\n    @llrule(loc, lambda parser: [])\n    def rule(parser):\n        return value\n    return rule", "response": "A rule that accepts no tokens and returns value."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Loc(kind, loc=None):\n    @llrule(loc, lambda parser: [kind])\n    def rule(parser):\n        result = parser._accept(kind)\n        if result is unmatched:\n            return result\n        return result.loc\n    return rule", "response": "A rule that accepts a token of kind and returns its location."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Expect(inner_rule, loc=None):\n    @llrule(loc, inner_rule.expected)\n    def rule(parser):\n        result = inner_rule(parser)\n        if result is unmatched:\n            expected = reduce(list.__add__, [rule.expected(parser) for rule in parser._errrules])\n            expected = list(sorted(set(expected)))\n\n            if len(expected) > 1:\n                expected = \" or \".join([\", \".join(expected[0:-1]), expected[-1]])\n            elif len(expected) == 1:\n                expected = expected[0]\n            else:\n                expected = \"(impossible)\"\n\n            error_tok = parser._tokens[parser._errindex]\n            error = diagnostic.Diagnostic(\n                \"fatal\", \"unexpected {actual}: expected {expected}\",\n                {\"actual\": error_tok.kind, \"expected\": expected},\n                error_tok.loc)\n            parser.diagnostic_engine.process(error)\n        return result\n    return rule", "response": "A rule that executes inner_rule and emits a diagnostic error if it returns None."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef SeqN(n, *inner_rules, **kwargs):\n    @action(Seq(*inner_rules), loc=kwargs.get(\"loc\", None))\n    def rule(parser, *values):\n        return values[n]\n    return rule", "response": "A rule that accepts a sequence of tokens satisfying rules and returns\n    the value returned by rule number n."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Alt(*inner_rules, **kwargs):\n    loc = kwargs.get(\"loc\", None)\n    expected = lambda parser: reduce(list.__add__, map(lambda x: x.expected(parser), inner_rules))\n    if loc is not None:\n        @llrule(loc, expected, cases=len(inner_rules))\n        def rule(parser):\n            data = parser._save()\n            for idx, inner_rule in enumerate(inner_rules):\n                result = inner_rule(parser)\n                if result is unmatched:\n                    parser._restore(data, rule=inner_rule)\n                else:\n                    rule.covered[idx] = True\n                    return result\n            return unmatched\n    else:\n        @llrule(loc, expected, cases=len(inner_rules))\n        def rule(parser):\n            data = parser._save()\n            for inner_rule in inner_rules:\n                result = inner_rule(parser)\n                if result is unmatched:\n                    parser._restore(data, rule=inner_rule)\n                else:\n                    return result\n            return unmatched\n    return rule", "response": "A rule that expects a sequence of tokens satisfying one of rules in sequence\n    and returns the return\n    value of that rule."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Newline(loc=None):\n    @llrule(loc, lambda parser: [\"newline\"])\n    def rule(parser):\n        result = parser._accept(\"newline\")\n        if result is unmatched:\n            return result\n        return []\n    return rule", "response": "A rule that accepts token of kind newline and returns an empty list."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef single_input(self, body):\n        loc = None\n        if body != []:\n            loc = body[0].loc\n        return ast.Interactive(body=body, loc=loc)", "response": "single_input is a wrapper around ast. Interactive"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef file_input(parser, body):\n        body = reduce(list.__add__, body, [])\n        loc = None\n        if body != []:\n            loc = body[0].loc\n        return ast.Module(body=body, loc=loc)", "response": "file_input - Parses the file input and returns a Module object"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef eval_input(self, expr):\n        return ast.Expression(body=[expr], loc=expr.loc)", "response": "eval_input: testlist NEWLINE * ENDMARKER"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef decorated(self, decorators, classfuncdef):\n        classfuncdef.at_locs = list(map(lambda x: x[0], decorators))\n        classfuncdef.decorator_list = list(map(lambda x: x[1], decorators))\n        classfuncdef.loc = classfuncdef.loc.join(decorators[0][0])\n        return classfuncdef", "response": "decorated - sets the at_locs and decorators of the classfuncdef"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef funcdef__26(self, def_loc, ident_tok, args, colon_loc, suite):\n        return ast.FunctionDef(name=ident_tok.value, args=args, returns=None,\n                               body=suite, decorator_list=[],\n                               at_locs=[], keyword_loc=def_loc, name_loc=ident_tok.loc,\n                               colon_loc=colon_loc, arrow_loc=None,\n                               loc=def_loc.join(suite[-1].loc))", "response": "(2.6, 2.7) funcdef: 'def' NAME parameters ':' suite"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef varargslist__26(self, fparams, args):\n        for fparam, default_opt in fparams:\n            if default_opt:\n                equals_loc, default = default_opt\n                args.equals_locs.append(equals_loc)\n                args.defaults.append(default)\n            elif len(args.defaults) > 0:\n                error = diagnostic.Diagnostic(\n                    \"fatal\", \"non-default argument follows default argument\", {},\n                    fparam.loc, [args.args[-1].loc.join(args.defaults[-1].loc)])\n                self.diagnostic_engine.process(error)\n\n            args.args.append(fparam)\n\n        def fparam_loc(fparam, default_opt):\n            if default_opt:\n                equals_loc, default = default_opt\n                return fparam.loc.join(default.loc)\n            else:\n                return fparam.loc\n\n        if args.loc is None:\n            args.loc = fparam_loc(*fparams[0]).join(fparam_loc(*fparams[-1]))\n        elif len(fparams) > 0:\n            args.loc = args.loc.join(fparam_loc(*fparams[0]))\n\n        return args", "response": "2. 6. 1. 2. 3. 2. 3. 2. 2."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nname [ : test ]", "response": "def tfpdef(self, ident_tok, annotation_opt):\n        \"\"\"(3.0-) tfpdef: NAME [':' test]\"\"\"\n        if annotation_opt:\n            colon_loc, annotation = annotation_opt\n            return self._arg(ident_tok, colon_loc, annotation)\n        return self._arg(ident_tok)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef expr_stmt(self, lhs, rhs):\n        if isinstance(rhs, ast.AugAssign):\n            if isinstance(lhs, ast.Tuple) or isinstance(lhs, ast.List):\n                error = diagnostic.Diagnostic(\n                    \"fatal\", \"illegal expression for augmented assignment\", {},\n                    rhs.op.loc, [lhs.loc])\n                self.diagnostic_engine.process(error)\n            else:\n                rhs.target = self._assignable(lhs)\n                rhs.loc = rhs.target.loc.join(rhs.value.loc)\n                return rhs\n        elif rhs is not None:\n            rhs.targets = list(map(self._assignable, [lhs] + rhs.targets))\n            rhs.loc = lhs.loc.join(rhs.value.loc)\n            return rhs\n        else:\n            return ast.Expr(value=lhs, loc=lhs.loc)", "response": "Return an expression statement that assigns lhs and rhs to the current object."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef del_stmt(self, stmt_loc, exprs):\n        # Python uses exprlist here, but does *not* obey the usual\n        # tuple-wrapping semantics, so we embed the rule directly.\n        \"\"\"del_stmt: 'del' exprlist\"\"\"\n        return ast.Delete(targets=[self._assignable(expr, is_delete=True) for expr in exprs],\n                          loc=stmt_loc.join(exprs[-1].loc), keyword_loc=stmt_loc)", "response": "Return an ast. Delete statement for the given list of expressions."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns statement for the cache entry.", "response": "def return_stmt(self, stmt_loc, values):\n        \"\"\"return_stmt: 'return' [testlist]\"\"\"\n        loc = stmt_loc\n        if values:\n            loc = loc.join(values.loc)\n        return ast.Return(value=values,\n                          loc=loc, keyword_loc=stmt_loc)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nyield the given expression.", "response": "def yield_stmt(self, expr):\n        \"\"\"yield_stmt: yield_expr\"\"\"\n        return ast.Expr(value=expr, loc=expr.loc)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef raise_stmt__30(self, raise_loc, exc_opt):\n        exc = from_loc = cause = None\n        loc = raise_loc\n        if exc_opt:\n            exc, cause_opt = exc_opt\n            loc = loc.join(exc.loc)\n            if cause_opt:\n                from_loc, cause = cause_opt\n                loc = loc.join(cause.loc)\n        return ast.Raise(exc=exc, inst=None, tback=None, cause=cause,\n                         keyword_loc=raise_loc, from_loc=from_loc, loc=loc)", "response": "raise statement for 3. 0"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef import_name(self, import_loc, names):\n        return ast.Import(names=names,\n                          keyword_loc=import_loc, loc=import_loc.join(names[-1].loc))", "response": "get_import_name - returns an ast. Import node"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef import_from(self, from_loc, module_name, import_loc, names):\n        (dots_loc, dots_count), dotted_name_opt = module_name\n        module_loc = module = None\n        if dotted_name_opt:\n            module_loc, module = dotted_name_opt\n        lparen_loc, names, rparen_loc = names\n        loc = from_loc.join(names[-1].loc)\n        if rparen_loc:\n            loc = loc.join(rparen_loc)\n\n        if module == \"__future__\":\n            self.add_flags([x.name for x in names])\n\n        return ast.ImportFrom(names=names, module=module, level=dots_count,\n                              keyword_loc=from_loc, dots_loc=dots_loc, module_loc=module_loc,\n                              import_loc=import_loc, lparen_loc=lparen_loc, rparen_loc=rparen_loc,\n                              loc=loc)", "response": "A function to create an ast. ImportFrom node."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nparses a global statement", "response": "def global_stmt(self, global_loc, names):\n        \"\"\"global_stmt: 'global' NAME (',' NAME)*\"\"\"\n        return ast.Global(names=list(map(lambda x: x.value, names)),\n                          name_locs=list(map(lambda x: x.loc, names)),\n                          keyword_loc=global_loc, loc=global_loc.join(names[-1].loc))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef exec_stmt(self, exec_loc, body, in_opt):\n        in_loc, globals, locals = None, None, None\n        loc = exec_loc.join(body.loc)\n        if in_opt:\n            in_loc, globals, locals = in_opt\n            if locals:\n                loc = loc.join(locals.loc)\n            else:\n                loc = loc.join(globals.loc)\n        return ast.Exec(body=body, locals=locals, globals=globals,\n                        loc=loc, keyword_loc=exec_loc, in_loc=in_loc)", "response": "exec_stmt is a statement that returns an ast. Exec instance."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef nonlocal_stmt(self, nonlocal_loc, names):\n        return ast.Nonlocal(names=list(map(lambda x: x.value, names)),\n                            name_locs=list(map(lambda x: x.loc, names)),\n                            keyword_loc=nonlocal_loc, loc=nonlocal_loc.join(names[-1].loc))", "response": "Return an ast. Nonlocal node for the given names."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning an ast. Assert node for the assert statement.", "response": "def assert_stmt(self, assert_loc, test, msg):\n        \"\"\"assert_stmt: 'assert' test [',' test]\"\"\"\n        loc = assert_loc.join(test.loc)\n        if msg:\n            loc = loc.join(msg.loc)\n        return ast.Assert(test=test, msg=msg,\n                          loc=loc, keyword_loc=assert_loc)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef if_stmt(self, if_loc, test, if_colon_loc, body, elifs, else_opt):\n        stmt = ast.If(orelse=[],\n                      else_loc=None, else_colon_loc=None)\n\n        if else_opt:\n            stmt.else_loc, stmt.else_colon_loc, stmt.orelse = else_opt\n\n        for elif_ in reversed(elifs):\n            stmt.keyword_loc, stmt.test, stmt.if_colon_loc, stmt.body = elif_\n            stmt.loc = stmt.keyword_loc.join(stmt.body[-1].loc)\n            if stmt.orelse:\n                stmt.loc = stmt.loc.join(stmt.orelse[-1].loc)\n            stmt = ast.If(orelse=[stmt],\n                          else_loc=None, else_colon_loc=None)\n\n        stmt.keyword_loc, stmt.test, stmt.if_colon_loc, stmt.body = \\\n            if_loc, test, if_colon_loc, body\n        stmt.loc = stmt.keyword_loc.join(stmt.body[-1].loc)\n        if stmt.orelse:\n            stmt.loc = stmt.loc.join(stmt.orelse[-1].loc)\n        return stmt", "response": "Parse an if statement."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef while_stmt(self, while_loc, test, while_colon_loc, body, else_opt):\n        stmt = ast.While(test=test, body=body, orelse=[],\n                         keyword_loc=while_loc, while_colon_loc=while_colon_loc,\n                         else_loc=None, else_colon_loc=None,\n                         loc=while_loc.join(body[-1].loc))\n        if else_opt:\n            stmt.else_loc, stmt.else_colon_loc, stmt.orelse = else_opt\n            stmt.loc = stmt.loc.join(stmt.orelse[-1].loc)\n\n        return stmt", "response": "parse a while statement"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef for_stmt(self, for_loc, target, in_loc, iter, for_colon_loc, body, else_opt):\n        stmt = ast.For(target=self._assignable(target), iter=iter, body=body, orelse=[],\n                       keyword_loc=for_loc, in_loc=in_loc, for_colon_loc=for_colon_loc,\n                       else_loc=None, else_colon_loc=None,\n                       loc=for_loc.join(body[-1].loc))\n        if else_opt:\n            stmt.else_loc, stmt.else_colon_loc, stmt.orelse = else_opt\n            stmt.loc = stmt.loc.join(stmt.orelse[-1].loc)\n\n        return stmt", "response": "for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef try_stmt(self, try_loc, try_colon_loc, body, stmt):\n        stmt.keyword_loc, stmt.try_colon_loc, stmt.body = \\\n            try_loc, try_colon_loc, body\n        stmt.loc = stmt.loc.join(try_loc)\n        return stmt", "response": "Parse a try statement."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef with_stmt__26(self, with_loc, context, with_var, colon_loc, body):\n        if with_var:\n            as_loc, optional_vars = with_var\n            item = ast.withitem(context_expr=context, optional_vars=optional_vars,\n                                as_loc=as_loc, loc=context.loc.join(optional_vars.loc))\n        else:\n            item = ast.withitem(context_expr=context, optional_vars=None,\n                                as_loc=None, loc=context.loc)\n        return ast.With(items=[item], body=body,\n                        keyword_loc=with_loc, colon_loc=colon_loc,\n                        loc=with_loc.join(body[-1].loc))", "response": "With statement 2. 6. 3. 1. 2."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef with_stmt__27(self, with_loc, items, colon_loc, body):\n        return ast.With(items=items, body=body,\n                        keyword_loc=with_loc, colon_loc=colon_loc,\n                        loc=with_loc.join(body[-1].loc))", "response": "With statement for base class 2. 7. 3. 1"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef old_lambdef(self, lambda_loc, args_opt, colon_loc, body):\n        if args_opt is None:\n            args_opt = self._arguments()\n            args_opt.loc = colon_loc.begin()\n        return ast.Lambda(args=args_opt, body=body,\n                          lambda_loc=lambda_loc, colon_loc=colon_loc,\n                          loc=lambda_loc.join(body.loc))", "response": "2. 6. 1. 2"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef comparison(self, lhs, rhs):\n        if len(rhs) > 0:\n            return ast.Compare(left=lhs, ops=list(map(lambda x: x[0], rhs)),\n                               comparators=list(map(lambda x: x[1], rhs)),\n                               loc=lhs.loc.join(rhs[-1][1].loc))\n        else:\n            return lhs", "response": "Compare two sets of entries."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef star_expr__30(self, star_opt, expr):\n        if star_opt:\n            return ast.Starred(value=expr, ctx=None,\n                               star_loc=star_opt, loc=expr.loc.join(star_opt))\n        return expr", "response": "(3.0, 3.1) star_expr: ['*'] expr"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef power(self, atom, trailers, factor_opt):\n        for trailer in trailers:\n            if isinstance(trailer, ast.Attribute) or isinstance(trailer, ast.Subscript):\n                trailer.value = atom\n            elif isinstance(trailer, ast.Call):\n                trailer.func = atom\n            trailer.loc = atom.loc.join(trailer.loc)\n            atom = trailer\n        if factor_opt:\n            op_loc, factor = factor_opt\n            return ast.BinOp(left=atom, op=ast.Pow(loc=op_loc), right=factor,\n                             loc=atom.loc.join(factor.loc))\n        return atom", "response": "power atom trailer * [ ** factor ]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget subscriptlist - returns subscript list", "response": "def subscriptlist(self, subscripts):\n        \"\"\"subscriptlist: subscript (',' subscript)* [',']\"\"\"\n        if len(subscripts) == 1:\n            return ast.Subscript(slice=subscripts[0], ctx=None, loc=None)\n        elif all([isinstance(x, ast.Index) for x in subscripts]):\n            elts  = [x.value for x in subscripts]\n            loc   = subscripts[0].loc.join(subscripts[-1].loc)\n            index = ast.Index(value=ast.Tuple(elts=elts, ctx=None,\n                                              begin_loc=None, end_loc=None, loc=loc),\n                              loc=loc)\n            return ast.Subscript(slice=index, ctx=None, loc=None)\n        else:\n            extslice = ast.ExtSlice(dims=subscripts,\n                                    loc=subscripts[0].loc.join(subscripts[-1].loc))\n            return ast.Subscript(slice=extslice, ctx=None, loc=None)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef dictmaker(self, elts):\n        return ast.Dict(keys=list(map(lambda x: x[0], elts)),\n                        values=list(map(lambda x: x[2], elts)),\n                        colon_locs=list(map(lambda x: x[1], elts)),\n                        loc=None)", "response": "(2.6) dictmaker: test ':' test (',' test ':' test)* [',']"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef arglist(self, args, call):\n        for arg in args:\n            if isinstance(arg, ast.keyword):\n                call.keywords.append(arg)\n            elif len(call.keywords) > 0:\n                error = diagnostic.Diagnostic(\n                    \"fatal\", \"non-keyword arg after keyword arg\", {},\n                    arg.loc, [call.keywords[-1].loc])\n                self.diagnostic_engine.process(error)\n            else:\n                call.args.append(arg)\n        return call", "response": "add arglist to call"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nyield expression 2. 6. 3. 2", "response": "def yield_expr__26(self, yield_loc, exprs):\n        \"\"\"(2.6, 2.7, 3.0, 3.1, 3.2) yield_expr: 'yield' [testlist]\"\"\"\n        if exprs is not None:\n            return ast.Yield(value=exprs,\n                             yield_loc=yield_loc, loc=yield_loc.join(exprs.loc))\n        else:\n            return ast.Yield(value=None,\n                             yield_loc=yield_loc, loc=yield_loc)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef urlparse(url, scheme='', allow_fragments=True):\n    tuple = urlsplit(url, scheme, allow_fragments)\n    scheme, netloc, url, query, fragment = tuple\n    if scheme in uses_params and ';' in url:\n        url, params = _splitparams(url)\n    else:\n        params = ''\n    return ParseResult(scheme, netloc, url, params, query, fragment)", "response": "Parse a URL into 6 components."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef urlsplit(url, scheme='', allow_fragments=True):\n    allow_fragments = bool(allow_fragments)\n    key = url, scheme, allow_fragments, type(url), type(scheme)\n    cached = _parse_cache.get(key, None)\n    if cached:\n        return cached\n    if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth\n        clear_cache()\n    netloc = query = fragment = ''\n    i = url.find(':')\n    if i > 0:\n        if url[:i] == 'http': # optimize the common case\n            scheme = url[:i].lower()\n            url = url[i+1:]\n            if url[:2] == '//':\n                netloc, url = _splitnetloc(url, 2)\n                if (('[' in netloc and ']' not in netloc) or\n                        (']' in netloc and '[' not in netloc)):\n                    raise ValueError(\"Invalid IPv6 URL\")\n            if allow_fragments and '#' in url:\n                url, fragment = url.split('#', 1)\n            if '?' in url:\n                url, query = url.split('?', 1)\n            v = SplitResult(scheme, netloc, url, query, fragment)\n            _parse_cache[key] = v\n            return v\n        for c in url[:i]:\n            if c not in scheme_chars:\n                break\n        else:\n            # make sure \"url\" is not actually a port number (in which case\n            # \"scheme\" is really part of the path)\n            rest = url[i+1:]\n            if not rest or any(c not in '0123456789' for c in rest):\n                # not a port number\n                scheme, url = url[:i].lower(), rest\n\n    if url[:2] == '//':\n        netloc, url = _splitnetloc(url, 2)\n        if (('[' in netloc and ']' not in netloc) or\n                (']' in netloc and '[' not in netloc)):\n            raise ValueError(\"Invalid IPv6 URL\")\n    if allow_fragments and '#' in url:\n        url, fragment = url.split('#', 1)\n    if '?' in url:\n        url, query = url.split('?', 1)\n    v = SplitResult(scheme, netloc, url, query, fragment)\n    _parse_cache[key] = v\n    return v", "response": "Parse a URL into 5 components."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nputting a parsed URL back together.", "response": "def urlunparse(data):\n    \"\"\"Put a parsed URL back together again.  This may result in a\n    slightly different, but equivalent URL, if the URL that was parsed\n    originally had redundant delimiters, e.g. a ? with an empty query\n    (the draft states that these are equivalent).\"\"\"\n    scheme, netloc, url, params, query, fragment = data\n    if params:\n        url = \"%s;%s\" % (url, params)\n    return urlunsplit((scheme, netloc, url, query, fragment))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef urlunsplit(data):\n    scheme, netloc, url, query, fragment = data\n    if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'):\n        if url and url[:1] != '/': url = '/' + url\n        url = '//' + (netloc or '') + url\n    if scheme:\n        url = scheme + ':' + url\n    if query:\n        url = url + '?' + query\n    if fragment:\n        url = url + '#' + fragment\n    return url", "response": "Combine the elements of a tuple as returned by urlsplit into a complete URL as a string."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\njoining a base URL and a possibly relative URL to form an absolute URL interpretation of the latter.", "response": "def urljoin(base, url, allow_fragments=True):\n    \"\"\"Join a base URL and a possibly relative URL to form an absolute\n    interpretation of the latter.\"\"\"\n    if not base:\n        return url\n    if not url:\n        return base\n    bscheme, bnetloc, bpath, bparams, bquery, bfragment = \\\n            urlparse(base, '', allow_fragments)\n    scheme, netloc, path, params, query, fragment = \\\n            urlparse(url, bscheme, allow_fragments)\n    if scheme != bscheme or scheme not in uses_relative:\n        return url\n    if scheme in uses_netloc:\n        if netloc:\n            return urlunparse((scheme, netloc, path,\n                               params, query, fragment))\n        netloc = bnetloc\n    if path[:1] == '/':\n        return urlunparse((scheme, netloc, path,\n                           params, query, fragment))\n    if not path and not params:\n        path = bpath\n        params = bparams\n        if not query:\n            query = bquery\n        return urlunparse((scheme, netloc, path,\n                           params, query, fragment))\n    segments = bpath.split('/')[:-1] + path.split('/')\n    # XXX The stuff below is bogus in various ways...\n    if segments[-1] == '.':\n        segments[-1] = ''\n    while '.' in segments:\n        segments.remove('.')\n    while 1:\n        i = 1\n        n = len(segments) - 1\n        while i < n:\n            if (segments[i] == '..'\n                and segments[i-1] not in ('', '..')):\n                del segments[i-1:i+1]\n                break\n            i = i+1\n        else:\n            break\n    if segments == ['', '..']:\n        segments[-1] = ''\n    elif len(segments) >= 2 and segments[-1] == '..':\n        segments[-2:] = ['']\n    return urlunparse((scheme, netloc, '/'.join(segments),\n                       params, query, fragment))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nremoving any existing fragment from the URL.", "response": "def urldefrag(url):\n    \"\"\"Removes any existing fragment from URL.\n\n    Returns a tuple of the defragmented URL and the fragment.  If\n    the URL contained no fragments, the second element is the\n    empty string.\n    \"\"\"\n    if '#' in url:\n        s, n, p, a, q, frag = urlparse(url)\n        defrag = urlunparse((s, n, p, a, q, ''))\n        return defrag, frag\n    else:\n        return url, ''"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing a query string given as a string argument.", "response": "def parse_qs(qs, keep_blank_values=0, strict_parsing=0):\n    \"\"\"Parse a query given as a string argument.\n\n        Arguments:\n\n        qs: percent-encoded query string to be parsed\n\n        keep_blank_values: flag indicating whether blank values in\n            percent-encoded queries should be treated as blank strings.\n            A true value indicates that blanks should be retained as\n            blank strings.  The default false value indicates that\n            blank values are to be ignored and treated as if they were\n            not included.\n\n        strict_parsing: flag indicating what to do with parsing errors.\n            If false (the default), errors are silently ignored.\n            If true, errors raise a ValueError exception.\n    \"\"\"\n    dict = {}\n    for name, value in parse_qsl(qs, keep_blank_values, strict_parsing):\n        if name in dict:\n            dict[name].append(value)\n        else:\n            dict[name] = [value]\n    return dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nparses a percent - encoded query string into a list of items.", "response": "def parse_qsl(qs, keep_blank_values=0, strict_parsing=0):\n    \"\"\"Parse a query given as a string argument.\n\n    Arguments:\n\n    qs: percent-encoded query string to be parsed\n\n    keep_blank_values: flag indicating whether blank values in\n        percent-encoded queries should be treated as blank strings.  A\n        true value indicates that blanks should be retained as blank\n        strings.  The default false value indicates that blank values\n        are to be ignored and treated as if they were  not included.\n\n    strict_parsing: flag indicating what to do with parsing errors. If\n        false (the default), errors are silently ignored. If true,\n        errors raise a ValueError exception.\n\n    Returns a list, as G-d intended.\n    \"\"\"\n    pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]\n    r = []\n    for name_value in pairs:\n        if not name_value and not strict_parsing:\n            continue\n        nv = name_value.split('=', 1)\n        if len(nv) != 2:\n            if strict_parsing:\n                raise ValueError, \"bad query field: %r\" % (name_value,)\n            # Handle case of a control-name with no equal sign\n            if keep_blank_values:\n                nv.append('')\n            else:\n                continue\n        if len(nv[1]) or keep_blank_values:\n            name = unquote(nv[0].replace('+', ' '))\n            value = unquote(nv[1].replace('+', ' '))\n            r.append((name, value))\n\n    return r"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _replace(_self, **kwds):\n        'Return a new SplitResult object replacing specified fields with new values'\n        result = _self._make(map(kwds.pop, ('scheme', 'netloc', 'path', 'query', 'fragment'), _self))\n        if kwds:\n            raise ValueError('Got unexpected field names: %r' % kwds.keys())\n        return result", "response": "Return a new SplitResult object replacing specified fields with new values"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the lines for a file from the cache.", "response": "def getlines(filename, module_globals=None):\n    \"\"\"Get the lines for a file from the cache.\n    Update the cache if it doesn't contain an entry for this file already.\"\"\"\n\n    if filename in cache:\n        return cache[filename][2]\n\n    try:\n        return updatecache(filename, module_globals)\n    except MemoryError:\n        clearcache()\n        return []"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef checkcache(filename=None):\n\n    if filename is None:\n        filenames = cache.keys()\n    else:\n        if filename in cache:\n            filenames = [filename]\n        else:\n            return\n\n    for filename in filenames:\n        size, mtime, lines, fullname = cache[filename]\n        if mtime is None:\n            continue   # no-op for files loaded via a __loader__\n        try:\n            stat = os.stat(fullname)\n        except os.error:\n            del cache[filename]\n            continue\n        if size != stat.st_size or mtime != stat.st_mtime:\n            del cache[filename]", "response": "Discard cache entries that are out of date."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef updatecache(filename, module_globals=None):\n\n    if filename in cache:\n        del cache[filename]\n    if not filename or (filename.startswith('<') and filename.endswith('>')):\n        return []\n\n    fullname = filename\n    try:\n        stat = os.stat(fullname)\n    except OSError:\n        basename = filename\n\n        # Try for a __loader__, if available\n        if module_globals and '__loader__' in module_globals:\n            name = module_globals.get('__name__')\n            loader = module_globals['__loader__']\n            get_source = getattr(loader, 'get_source', None)\n\n            if name and get_source:\n                try:\n                    data = get_source(name)\n                except (ImportError, IOError):\n                    pass\n                else:\n                    if data is None:\n                        # No luck, the PEP302 loader cannot find the source\n                        # for this module.\n                        return []\n                    cache[filename] = (\n                        len(data), None,\n                        [line+'\\n' for line in data.splitlines()], fullname\n                    )\n                    return cache[filename][2]\n\n        # Try looking through the module search path, which is only useful\n        # when handling a relative filename.\n        if os.path.isabs(filename):\n            return []\n\n        for dirname in sys.path:\n            # When using imputil, sys.path may contain things other than\n            # strings; ignore them when it happens.\n            try:\n                fullname = os.path.join(dirname, basename)\n            except (TypeError, AttributeError):\n                # Not sufficiently string-like to do anything useful with.\n                continue\n            try:\n                stat = os.stat(fullname)\n                break\n            except os.error:\n                pass\n        else:\n            return []\n    try:\n        with open(fullname, 'rU') as fp:\n            lines = fp.readlines()\n    except IOError:\n        return []\n    if lines and not lines[-1].endswith('\\n'):\n        lines[-1] += '\\n'\n    size, mtime = stat.st_size, stat.st_mtime\n    cache[filename] = size, mtime, lines, fullname\n    return lines", "response": "Update a cache entry and return its list of lines."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef isfile(path):\n    try:\n        st = os.stat(path)\n    except os.error:\n        return False\n    return stat.S_ISREG(st.st_mode)", "response": "Test whether a path is a regular file"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning True if the pathname refers to an existing directory.", "response": "def isdir(s):\n    \"\"\"Return true if the pathname refers to an existing directory.\"\"\"\n    try:\n        st = os.stat(s)\n    except os.error:\n        return False\n    return stat.S_ISDIR(st.st_mode)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef commonprefix(m):\n    \"Given a list of pathnames, returns the longest common leading component\"\n    if not m: return ''\n    s1 = min(m)\n    s2 = max(m)\n    for i, c in enumerate(s1):\n        if c != s2[i]:\n            return s1[:i]\n    return s1", "response": "Given a list of pathnames returns the longest common leading component"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _splitext(p, sep, altsep, extsep):\n\n    sepIndex = p.rfind(sep)\n    if altsep:\n        altsepIndex = p.rfind(altsep)\n        sepIndex = max(sepIndex, altsepIndex)\n\n    dotIndex = p.rfind(extsep)\n    if dotIndex > sepIndex:\n        # skip all leading dots\n        filenameIndex = sepIndex + 1\n        while filenameIndex < dotIndex:\n            if p[filenameIndex] != extsep:\n                return p[:dotIndex], p[dotIndex:]\n            filenameIndex += 1\n\n    return p, ''", "response": "Split the extension from a pathname."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nwraps a single paragraph of text returning a list of wrapped lines.", "response": "def wrap(text, width=70, **kwargs):\n    \"\"\"Wrap a single paragraph of text, returning a list of wrapped lines.\n\n    Reformat the single paragraph in 'text' so it fits in lines of no\n    more than 'width' columns, and return a list of wrapped lines.  By\n    default, tabs in 'text' are expanded with string.expandtabs(), and\n    all other whitespace characters (including newline) are converted to\n    space.  See TextWrapper class for available keyword args to customize\n    wrapping behaviour.\n    \"\"\"\n    w = TextWrapper(width=width, **kwargs)\n    return w.wrap(text)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef fill(text, width=70, **kwargs):\n    w = TextWrapper(width=width, **kwargs)\n    return w.fill(text)", "response": "Fill a single paragraph of text returning a new string."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef dedent(text):\n    # Look for the longest leading string of spaces and tabs common to\n    # all lines.\n    margin = None\n    text = _whitespace_only_re.sub('', text)\n    indents = _leading_whitespace_re.findall(text)\n    for indent in indents:\n        if margin is None:\n            margin = indent\n\n        # Current line more deeply indented than previous winner:\n        # no change (previous winner is still on top).\n        elif indent.startswith(margin):\n            pass\n\n        # Current line consistent with and no deeper than previous winner:\n        # it's the new winner.\n        elif margin.startswith(indent):\n            margin = indent\n\n        # Find the largest common whitespace between current line and previous\n        # winner.\n        else:\n            for i, (x, y) in enumerate(zip(margin, indent)):\n                if x != y:\n                    margin = margin[:i]\n                    break\n            else:\n                margin = margin[:len(indent)]\n\n    # sanity check (testing/debugging only)\n    if 0 and margin:\n        for line in text.split(\"\\n\"):\n            assert not line or line.startswith(margin), \\\n                   \"line = %r, margin = %r\" % (line, margin)\n\n    if margin:\n        text = re.sub(r'(?m)^' + margin, '', text)\n    return text", "response": "Remove any common leading whitespace from every line in text."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _split(self, text):\n        if isinstance(text, _unicode):\n            if self.break_on_hyphens:\n                pat = self.wordsep_re_uni\n            else:\n                pat = self.wordsep_simple_re_uni\n        else:\n            if self.break_on_hyphens:\n                pat = self.wordsep_re\n            else:\n                pat = self.wordsep_simple_re\n        chunks = pat.split(text)\n        # chunks = filter(None, chunks)  # remove empty chunks\n        chunks = [x for x in chunks if x is not None]\n        return chunks", "response": "Split the text into individual words."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nfix for sentence endings buried in chunks.", "response": "def _fix_sentence_endings(self, chunks):\n        \"\"\"_fix_sentence_endings(chunks : [string])\n\n        Correct for sentence endings buried in 'chunks'.  Eg. when the\n        original text contains \"... foo.\\\\nBar ...\", munge_whitespace()\n        and split() will convert that to [..., \"foo.\", \" \", \"Bar\", ...]\n        which has one too few spaces; this method simply changes the one\n        space to two.\n        \"\"\"\n        i = 0\n        patsearch = self.sentence_end_re.search\n        while i < len(chunks)-1:\n            if chunks[i+1] == \" \" and patsearch(chunks[i]):\n                chunks[i+1] = \"  \"\n                i += 2\n            else:\n                i += 1"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nhandle a long word.", "response": "def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):\n        \"\"\"_handle_long_word(chunks : [string],\n                             cur_line : [string],\n                             cur_len : int, width : int)\n\n        Handle a chunk of text (most likely a word, not whitespace) that\n        is too long to fit in any line.\n        \"\"\"\n        # Figure out when indent is larger than the specified width, and make\n        # sure at least one character is stripped off on every pass\n        if width < 1:\n            space_left = 1\n        else:\n            space_left = width - cur_len\n\n        # If we're allowed to break long words, then do so: put as much\n        # of the next chunk onto the current line as will fit.\n        if self.break_long_words:\n            cur_line.append(reversed_chunks[-1][:space_left])\n            reversed_chunks[-1] = reversed_chunks[-1][space_left:]\n\n        # Otherwise, we have to preserve the long word intact.  Only add\n        # it to the current line if there's nothing already there --\n        # that minimizes how much we violate the width constraint.\n        elif not cur_line:\n            cur_line.append(reversed_chunks.pop())"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _wrap_chunks(self, chunks):\n        lines = []\n        if self.width <= 0:\n            raise ValueError(\"invalid width %r (must be > 0)\" % self.width)\n\n        # Arrange in reverse order so items can be efficiently popped\n        # from a stack of chucks.\n        chunks.reverse()\n\n        while chunks:\n\n            # Start the list of chunks that will make up the current line.\n            # cur_len is just the length of all the chunks in cur_line.\n            cur_line = []\n            cur_len = 0\n\n            # Figure out which static string will prefix this line.\n            if lines:\n                indent = self.subsequent_indent\n            else:\n                indent = self.initial_indent\n\n            # Maximum width for this line.\n            width = self.width - len(indent)\n\n            # First chunk on line is whitespace -- drop it, unless this\n            # is the very beginning of the text (ie. no lines started yet).\n            if self.drop_whitespace and chunks[-1].strip() == '' and lines:\n                # del chunks[-1]\n                chunks.pop()\n\n            while chunks:\n                l = len(chunks[-1])\n\n                # Can at least squeeze this chunk onto the current line.\n                if cur_len + l <= width:\n                    cur_line.append(chunks.pop())\n                    cur_len += l\n\n                # Nope, this line is full.\n                else:\n                    break\n\n            # The current line is full, and the next chunk is too big to\n            # fit on *any* line (not just this one).\n            if chunks and len(chunks[-1]) > width:\n                self._handle_long_word(chunks, cur_line, cur_len, width)\n\n            # If the last chunk on this line is all whitespace, drop it.\n            if self.drop_whitespace and cur_line and cur_line[-1].strip() == '':\n                # del cur_line[-1]\n                cur_line.pop()\n\n            # Convert current line back to a string and store it in list\n            # of all lines (return value).\n            if cur_line:\n                lines.append(indent + ''.join(cur_line))\n\n        return lines", "response": "Wrap a sequence of text chunks into a list of lines of length self. width."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef wrap(self, text):\n        text = self._munge_whitespace(text)\n        chunks = self._split(text)\n        if self.fix_sentence_endings:\n            self._fix_sentence_endings(chunks)\n        return self._wrap_chunks(chunks)", "response": "Wraps the given text into a list of lines of the specified width."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _long2bytesBigEndian(n, blocksize=0):\n\n    # After much testing, this algorithm was deemed to be the fastest.\n    s = b''\n    pack = struct.pack\n    while n > 0:\n        s = pack('>I', n & 0xffffffff) + s\n        n = n >> 32\n\n    # Strip off leading zeros.\n    for i in range(len(s)):\n        if s[i] != '\\000':\n            break\n    else:\n        # Only happens when n == 0.\n        s = '\\000'\n        i = 0\n\n    s = s[i:]\n\n    # Add back some pad bytes. This could be done more efficiently\n    # w.r.t. the de-padding being done above, but sigh...\n    if blocksize > 0 and len(s) % blocksize:\n        s = (blocksize - len(s) % blocksize) * '\\000' + s\n\n    return s", "response": "Convert a long integer to a byte string in big - endian byte order."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntransform a list of characters into a list of longs.", "response": "def _bytelist2longBigEndian(list):\n    \"Transform a list of characters into a list of longs.\"\n\n    imax = len(list) // 4\n    hl = [0] * imax\n\n    j = 0\n    i = 0\n    while i < imax:\n        b0 = ord(list[j]) << 24\n        b1 = ord(list[j+1]) << 16\n        b2 = ord(list[j+2]) << 8\n        b3 = ord(list[j+3])\n        hl[i] = b0 | b1 | b2 | b3\n        i = i+1\n        j = j+4\n\n    return hl"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing the message - digest and set all fields to zero.", "response": "def init(self):\n        \"Initialize the message-digest and set all fields to zero.\"\n\n        self.length = 0\n        self.input = []\n\n        # Initial 160 bit message digest (5 times 32 bit).\n        self.H0 = 0x67452301\n        self.H1 = 0xEFCDAB89\n        self.H2 = 0x98BADCFE\n        self.H3 = 0x10325476\n        self.H4 = 0xC3D2E1F0"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nterminates the message - digest computation and return digest.", "response": "def digest(self):\n        \"\"\"Terminate the message-digest computation and return digest.\n\n        Return the digest of the strings passed to the update()\n        method so far. This is a 16-byte string which may contain\n        non-ASCII characters, including null bytes.\n        \"\"\"\n\n        H0 = self.H0\n        H1 = self.H1\n        H2 = self.H2\n        H3 = self.H3\n        H4 = self.H4\n        input = [] + self.input\n        count = [] + self.count\n\n        index = (self.count[1] >> 3) & 0x3f\n\n        if index < 56:\n            padLen = 56 - index\n        else:\n            padLen = 120 - index\n\n        padding = ['\\200'] + ['\\000'] * 63\n        self.update(padding[:padLen])\n\n        # Append length (before padding).\n        bits = _bytelist2longBigEndian(self.input[:56]) + count\n\n        self._transform(bits)\n\n        # Store state in digest.\n        digest = _long2bytesBigEndian(self.H0, 4) + \\\n                 _long2bytesBigEndian(self.H1, 4) + \\\n                 _long2bytesBigEndian(self.H2, 4) + \\\n                 _long2bytesBigEndian(self.H3, 4) + \\\n                 _long2bytesBigEndian(self.H4, 4)\n\n        self.H0 = H0\n        self.H1 = H1\n        self.H2 = H2\n        self.H3 = H3\n        self.H4 = H4\n        self.input = input\n        self.count = count\n\n        return digest"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the next token in the queue.", "response": "def next(self, eof_token=False):\n        \"\"\"\n        Returns token at ``offset`` as a :class:`Token` and advances ``offset``\n        to point past the end of the token, where the token has:\n\n        - *range* which is a :class:`pythonparser.source.Range` that includes\n          the token but not surrounding whitespace,\n        - *kind* which is a string containing one of Python keywords or operators,\n          ``newline``, ``float``, ``int``, ``complex``, ``strbegin``,\n          ``strdata``, ``strend``, ``ident``, ``indent``, ``dedent`` or ``eof``\n          (if ``eof_token`` is True).\n        - *value* which is the flags as lowercase string if *kind* is ``strbegin``,\n          the string contents if *kind* is ``strdata``,\n          the numeric value if *kind* is ``float``, ``int`` or ``complex``,\n          the identifier if *kind* is ``ident`` and ``None`` in any other case.\n\n        :param eof_token: if true, will return a token with kind ``eof``\n            when the input is exhausted; if false, will raise ``StopIteration``.\n        \"\"\"\n        if len(self.queue) == 0:\n            self._refill(eof_token)\n\n        return self.queue.pop(0)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef enterabs(self, time, priority, action, argument):\n        event = Event(time, priority, action, argument)\n        heapq.heappush(self._queue, event)\n        return event", "response": "Enter a new event at an absolute time."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef run(self):\n        # localize variable access to minimize overhead\n        # and to improve thread safety\n        q = self._queue\n        delayfunc = self.delayfunc\n        timefunc = self.timefunc\n        pop = heapq.heappop\n        while q:\n            # TODO: modified part of grumpy version.\n            checked_event = q[0]\n            time, priority, action, argument = checked_event.get_fields()\n            now = timefunc()\n            if now < time:\n                delayfunc(time - now)\n            else:\n                event = pop(q)\n                # Verify that the event was not removed or altered\n                # by another thread after we last looked at q[0].\n                if event is checked_event:\n                    action(*argument)\n                    delayfunc(0)   # Let other threads run\n                else:\n                    heapq.heappush(q, event)", "response": "Execute events until the queue is empty and the scheduler is running."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nregistering a new dialect.", "response": "def register_dialect(name, dialect=None, **kwargs):\n    \"\"\"Create a mapping from a string name to a dialect class.\n    dialect = csv.register_dialect(name, dialect)\"\"\"\n    if not isinstance(name, basestring):\n        raise TypeError(\"dialect name must be a string or unicode\")\n\n    dialect = _call_dialect(dialect, kwargs)\n    _dialects[name] = dialect"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef field_size_limit(limit=undefined):\n\n    global _field_limit\n    old_limit = _field_limit\n\n    if limit is not undefined:\n        if not isinstance(limit, (int, long)):\n            raise TypeError(\"int expected, got %s\" %\n                            (limit.__class__.__name__,))\n        _field_limit = limit\n\n    return old_limit", "response": "Sets the upper limit on parsed fields.\n    Returns old limit"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nshallows copy operation on arbitrary Python objects.", "response": "def copy(x):\n    \"\"\"Shallow copy operation on arbitrary Python objects.\n\n    See the module's __doc__ string for more info.\n    \"\"\"\n\n    cls = type(x)\n\n    copier = _copy_dispatch.get(cls)\n    if copier:\n        return copier(x)\n\n    copier = getattr(cls, \"__copy__\", None)\n    if copier:\n        return copier(x)\n\n    reductor = dispatch_table.get(cls)\n    if reductor:\n        rv = reductor(x)\n    else:\n        reductor = getattr(x, \"__reduce_ex__\", None)\n        if reductor:\n            rv = reductor(2)\n        else:\n            reductor = getattr(x, \"__reduce__\", None)\n            if reductor:\n                rv = reductor()\n            else:\n                raise Error(\"un(shallow)copyable object of type %s\" % cls)\n\n    return _reconstruct(x, rv, 0)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef deepcopy(x, memo=None, _nil=[]):\n\n    if memo is None:\n        memo = {}\n\n    d = id(x)\n    y = memo.get(d, _nil)\n    if y is not _nil:\n        return y\n\n    cls = type(x)\n\n    copier = _deepcopy_dispatch.get(cls)\n    if copier:\n        y = copier(x, memo)\n    else:\n        try:\n            issc = issubclass(cls, type)\n        except TypeError: # cls is not a class (old Boost; see SF #502085)\n            issc = 0\n        if issc:\n            y = _deepcopy_atomic(x, memo)\n        else:\n            copier = getattr(x, \"__deepcopy__\", None)\n            if copier:\n                y = copier(memo)\n            else:\n                reductor = dispatch_table.get(cls)\n                if reductor:\n                    rv = reductor(x)\n                else:\n                    reductor = getattr(x, \"__reduce_ex__\", None)\n                    if reductor:\n                        rv = reductor(2)\n                    else:\n                        reductor = getattr(x, \"__reduce__\", None)\n                        if reductor:\n                            rv = reductor()\n                        else:\n                            raise Error(\n                                \"un(deep)copyable object of type %s\" % cls)\n                y = _reconstruct(x, rv, 1, memo)\n\n    memo[d] = y\n    _keep_alive(x, memo) # Make sure x lives at least as long as d\n    return y", "response": "Deep copy operation on arbitrary Python objects."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nkeep a reference to the object x in the memo.", "response": "def _keep_alive(x, memo):\n    \"\"\"Keeps a reference to the object x in the memo.\n\n    Because we remember objects by their id, we have\n    to assure that possibly temporary objects are kept\n    alive by referencing them.\n    We store a reference at the id of the memo, which should\n    normally not be used unless someone tries to deepcopy\n    the memo itself...\n    \"\"\"\n    try:\n        memo[id(memo)].append(x)\n    except KeyError:\n        # aha, this is the first one :-)\n        memo[id(memo)]=[x]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef warnpy3k(message, category=None, stacklevel=1):\n    if sys.py3kwarning:\n        if category is None:\n            category = DeprecationWarning\n        warn(message, category, stacklevel+1)", "response": "Issue a deprecation warning for Python 3. x related changes."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nhooks to write a warning to a file ; replace if you like.", "response": "def _show_warning(message, category, filename, lineno, file=None, line=None):\n    \"\"\"Hook to write a warning to a file; replace if you like.\"\"\"\n    if file is None:\n        file = sys.stderr\n        if file is None:\n            # sys.stderr is None - warnings get lost\n            return\n    try:\n        file.write(formatwarning(message, category, filename, lineno, line))\n    except (IOError, UnicodeError):\n        pass # the file (probably stderr) is invalid - this warning gets lost."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nfunctions to format a warning the standard way.", "response": "def formatwarning(message, category, filename, lineno, line=None):\n    \"\"\"Function to format a warning the standard way.\"\"\"\n    try:\n        unicodetype = unicode\n    except NameError:\n        unicodetype = ()\n    try:\n        message = str(message)\n    except UnicodeEncodeError:\n        pass\n    s =  \"%s: %s: %s\\n\" % (lineno, category.__name__, message)\n    line = linecache.getline(filename, lineno) if line is None else line\n    if line:\n        line = line.strip()\n        if isinstance(s, unicodetype) and isinstance(line, str):\n            line = unicode(line, 'latin1')\n        s += \"  %s\\n\" % line\n    if isinstance(s, unicodetype) and isinstance(filename, str):\n        enc = sys.getfilesystemencoding()\n        if enc:\n            try:\n                filename = unicode(filename, enc)\n            except UnicodeDecodeError:\n                pass\n    s = \"%s:%s\" % (filename, s)\n    return s"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nadding a new entry to the list of filters that match the given message.", "response": "def filterwarnings(action, message=\"\", category=Warning, module=\"\", lineno=0,\n                   append=0):\n    \"\"\"Insert an entry into the list of warnings filters (at the front).\n\n    'action' -- one of \"error\", \"ignore\", \"always\", \"default\", \"module\",\n                or \"once\"\n    'message' -- a regex that the warning message must match\n    'category' -- a class that the warning must be a subclass of\n    'module' -- a regex that the module name must match\n    'lineno' -- an integer line number, 0 matches all warnings\n    'append' -- if true, append to the list of filters\n    \"\"\"\n    assert action in (\"error\", \"ignore\", \"always\", \"default\", \"module\",\n                      \"once\"), \"invalid action: %r\" % (action,)\n    assert isinstance(message, basestring), \"message must be a string\"\n    assert isinstance(category, type), \"category must be a class\"\n    assert issubclass(category, Warning), \"category must be a Warning subclass\"\n    assert isinstance(module, basestring), \"module must be a string\"\n    assert isinstance(lineno, int) and lineno >= 0, \\\n           \"lineno must be an int >= 0\"\n    item = (action, re.compile(message, re.I), category,\n            re.compile(module), lineno)\n    if append:\n        filters.append(item)\n    else:\n        filters.insert(0, item)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ninserts a simple filter into the list of filters.", "response": "def simplefilter(action, category=Warning, lineno=0, append=0):\n    \"\"\"Insert a simple entry into the list of warnings filters (at the front).\n\n    A simple filter matches all modules and messages.\n    'action' -- one of \"error\", \"ignore\", \"always\", \"default\", \"module\",\n                or \"once\"\n    'category' -- a class that the warning must be a subclass of\n    'lineno' -- an integer line number, 0 matches all warnings\n    'append' -- if true, append to the list of filters\n    \"\"\"\n    assert action in (\"error\", \"ignore\", \"always\", \"default\", \"module\",\n                      \"once\"), \"invalid action: %r\" % (action,)\n    assert isinstance(lineno, int) and lineno >= 0, \\\n           \"lineno must be an int >= 0\"\n    item = (action, None, category, None, lineno)\n    if append:\n        filters.append(item)\n    else:\n        filters.insert(0, item)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nissue a warning or maybe raise an exception.", "response": "def warn(message, category=None, stacklevel=1):\n    \"\"\"Issue a warning, or maybe ignore it or raise an exception.\"\"\"\n    # Check if message is already a Warning object\n    if isinstance(message, Warning):\n        category = message.__class__\n    # Check category argument\n    if category is None:\n        category = UserWarning\n    assert issubclass(category, Warning)\n    # Get context information\n    try:\n        caller = sys._getframe(stacklevel)\n    except ValueError:\n        globals = sys.__dict__\n        lineno = 1\n    else:\n        globals = caller.f_globals\n        lineno = caller.f_lineno\n    if '__name__' in globals:\n        module = globals['__name__']\n    else:\n        module = \"<string>\"\n    filename = globals.get('__file__')\n    if filename:\n        fnl = filename.lower()\n        if fnl.endswith((\".pyc\", \".pyo\")):\n            filename = filename[:-1]\n    else:\n        if module == \"__main__\":\n            try:\n                filename = sys.argv[0]\n            except AttributeError:\n                # embedded interpreters don't have sys.argv, see bug #839151\n                filename = '__main__'\n        if not filename:\n            filename = module\n    registry = globals.setdefault(\"__warningregistry__\", {})\n    warn_explicit(message, category, filename, lineno, module, registry,\n                  globals)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _hash(self):\n        MAX = sys.maxint\n        MASK = 2 * MAX + 1\n        n = len(self)\n        h = 1927868237 * (n + 1)\n        h &= MASK\n        for x in self:\n            hx = hash(x)\n            h ^= (hx ^ (hx << 16) ^ 89869747)  * 3644798167\n            h &= MASK\n        h = h * 69069 + 907133923\n        h &= MASK\n        if h > MAX:\n            h -= MASK + 1\n        if h == -1:\n            h = 590923713\n        return h", "response": "Compute the hash value of a set of items."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nremoves an element from the set.", "response": "def remove(self, value):\n        \"\"\"Remove an element. If not a member, raise a KeyError.\"\"\"\n        if value not in self:\n            raise KeyError(value)\n        self.discard(value)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef pop(self):\n        it = iter(self)\n        try:\n            value = next(it)\n        except StopIteration:\n            raise KeyError\n        self.discard(value)\n        return value", "response": "Return the popped value. Raise KeyError if empty."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update(*args, **kwds):\n        ''' D.update([E, ]**F) -> None.  Update D from mapping/iterable E and F.\n            If E present and has a .keys() method, does:     for k in E: D[k] = E[k]\n            If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v\n            In either case, this is followed by: for k, v in F.items(): D[k] = v\n        '''\n        if not args:\n            raise TypeError(\"descriptor 'update' of 'MutableMapping' object \"\n                            \"needs an argument\")\n        self = args[0]\n        args = args[1:]\n        if len(args) > 1:\n            raise TypeError('update expected at most 1 arguments, got %d' %\n                            len(args))\n        if args:\n            other = args[0]\n            if isinstance(other, Mapping):\n                for key in other:\n                    self[key] = other[key]\n            elif hasattr(other, \"keys\"):\n                for key in other.keys():\n                    self[key] = other[key]\n            else:\n                for key, value in other:\n                    self[key] = value\n        for key, value in kwds.items():\n            self[key] = value", "response": "Update the object with the contents of the object."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef index(self, value):\n        '''S.index(value) -> integer -- return first index of value.\n           Raises ValueError if the value is not present.\n        '''\n        for i, v in enumerate(self):\n            if v == value:\n                return i\n        raise ValueError", "response": "S. index - Returns the first index of value."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreverse the list of all the elements of the array.", "response": "def reverse(self):\n        'S.reverse() -- reverse *IN PLACE*'\n        n = len(self)\n        for i in range(n//2):\n            self[i], self[n-i-1] = self[n-i-1], self[i]"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef go_str(value):\n  io = StringIO.StringIO()\n  io.write('\"')\n  for c in value:\n    if c in _ESCAPES:\n      io.write(_ESCAPES[c])\n    elif c in _SIMPLE_CHARS:\n      io.write(c)\n    else:\n      io.write(r'\\x{:02x}'.format(ord(c)))\n  io.write('\"')\n  return io.getvalue()", "response": "Returns value as a valid Go string literal."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef write_block(self, block_, body):\n    self.write('for ; \u03c0F.State() >= 0; \u03c0F.PopCheckpoint() {')\n    with self.indent_block():\n      self.write('switch \u03c0F.State() {')\n      self.write('case 0:')\n      for checkpoint in block_.checkpoints:\n        self.write_tmpl('case $state: goto Label$state', state=checkpoint)\n      self.write('default: panic(\"unexpected function state\")')\n      self.write('}')\n      # Assume that body is aligned with goto labels.\n      with self.indent_block(-1):\n        self.write(body)\n    self.write('}')", "response": "Outputs the boilerplate needed for code blocks like functions."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef acquire(self, blocking=1):\n        me = _get_ident()\n        if self.__owner == me:\n            self.__count = self.__count + 1\n            if __debug__:\n                self._note(\"%s.acquire(%s): recursive success\", self, blocking)\n            return 1\n        rc = self.__block.acquire(blocking)\n        if rc:\n            self.__owner = me\n            self.__count = 1\n            if __debug__:\n                self._note(\"%s.acquire(%s): initial success\", self, blocking)\n        else:\n            if __debug__:\n                self._note(\"%s.acquire(%s): failure\", self, blocking)\n        return rc", "response": "Acquire a lock, blocking or non-blocking.\n\n        When invoked without arguments: if this thread already owns the lock,\n        increment the recursion level by one, and return immediately. Otherwise,\n        if another thread owns the lock, block until the lock is unlocked. Once\n        the lock is unlocked (not owned by any thread), then grab ownership, set\n        the recursion level to one, and return. If more than one thread is\n        blocked waiting until the lock is unlocked, only one at a time will be\n        able to grab ownership of the lock. There is no return value in this\n        case.\n\n        When invoked with the blocking argument set to true, do the same thing\n        as when called without arguments, and return true.\n\n        When invoked with the blocking argument set to false, do not block. If a\n        call without an argument would block, return false immediately;\n        otherwise, do the same thing as when called without arguments, and\n        return true."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef release(self):\n        if self.__owner != _get_ident():\n            raise RuntimeError(\"cannot release un-acquired lock\")\n        self.__count = count = self.__count - 1\n        if not count:\n            self.__owner = None\n            self.__block.release()\n            if __debug__:\n                self._note(\"%s.release(): final release\", self)\n        else:\n            if __debug__:\n                self._note(\"%s.release(): non-final release\", self)", "response": "Release a lock, decrementing the recursion level.\n\n        If after the decrement it is zero, reset the lock to unlocked (not owned\n        by any thread), and if any other threads are blocked waiting for the\n        lock to become unlocked, allow exactly one of them to proceed. If after\n        the decrement the recursion level is still nonzero, the lock remains\n        locked and owned by the calling thread.\n\n        Only call this method when the calling thread owns the lock. A\n        RuntimeError is raised if this method is called when the lock is\n        unlocked.\n\n        There is no return value."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef wait(self, timeout=None):\n        if not self._is_owned():\n            raise RuntimeError(\"cannot wait on un-acquired lock\")\n        waiter = _allocate_lock()\n        waiter.acquire()\n        self.__waiters.append(waiter)\n        saved_state = self._release_save()\n        try:    # restore state no matter what (e.g., KeyboardInterrupt)\n            if timeout is None:\n                waiter.acquire()\n                if __debug__:\n                    self._note(\"%s.wait(): got it\", self)\n            else:\n                # Balancing act:  We can't afford a pure busy loop, so we\n                # have to sleep; but if we sleep the whole timeout time,\n                # we'll be unresponsive.  The scheme here sleeps very\n                # little at first, longer as time goes on, but never longer\n                # than 20 times per second (or the timeout time remaining).\n                endtime = _time() + timeout\n                delay = 0.0005 # 500 us -> initial delay of 1 ms\n                while True:\n                    gotit = waiter.acquire(0)\n                    if gotit:\n                        break\n                    remaining = endtime - _time()\n                    if remaining <= 0:\n                        break\n                    delay = min(delay * 2, remaining, .05)\n                    _sleep(delay)\n                if not gotit:\n                    if __debug__:\n                        self._note(\"%s.wait(%s): timed out\", self, timeout)\n                    try:\n                        self.__waiters.remove(waiter)\n                    except ValueError:\n                        pass\n                else:\n                    if __debug__:\n                        self._note(\"%s.wait(%s): got it\", self, timeout)\n        finally:\n            self._acquire_restore(saved_state)", "response": "Wait until the current thread is notified or until a timeout occurs."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef notify(self, n=1):\n        if not self._is_owned():\n            raise RuntimeError(\"cannot notify on un-acquired lock\")\n        __waiters = self.__waiters\n        waiters = __waiters[:n]\n        if not waiters:\n            if __debug__:\n                self._note(\"%s.notify(): no waiters\", self)\n            return\n        self._note(\"%s.notify(): notifying %d waiter%s\", self, n,\n                   n!=1 and \"s\" or \"\")\n        for waiter in waiters:\n            waiter.release()\n            try:\n                __waiters.remove(waiter)\n            except ValueError:\n                pass", "response": "Wake up at most n threads waiting on this condition."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nacquiring a semaphore of the current value.", "response": "def acquire(self, blocking=1):\n        \"\"\"Acquire a semaphore, decrementing the internal counter by one.\n\n        When invoked without arguments: if the internal counter is larger than\n        zero on entry, decrement it by one and return immediately. If it is zero\n        on entry, block, waiting until some other thread has called release() to\n        make it larger than zero. This is done with proper interlocking so that\n        if multiple acquire() calls are blocked, release() will wake exactly one\n        of them up. The implementation may pick one at random, so the order in\n        which blocked threads are awakened should not be relied on. There is no\n        return value in this case.\n\n        When invoked with blocking set to true, do the same thing as when called\n        without arguments, and return true.\n\n        When invoked with blocking set to false, do not block. If a call without\n        an argument would block, return false immediately; otherwise, do the\n        same thing as when called without arguments, and return true.\n\n        \"\"\"\n        rc = False\n        with self.__cond:\n            while self.__value == 0:\n                if not blocking:\n                    break\n                if __debug__:\n                    self._note(\"%s.acquire(%s): blocked waiting, value=%s\",\n                            self, blocking, self.__value)\n                self.__cond.wait()\n            else:\n                self.__value = self.__value - 1\n                if __debug__:\n                    self._note(\"%s.acquire: success, value=%s\",\n                            self, self.__value)\n                rc = True\n        return rc"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef release(self):\n        with self.__cond:\n            self.__value = self.__value + 1\n            if __debug__:\n                self._note(\"%s.release: success, value=%s\",\n                        self, self.__value)\n            self.__cond.notify()", "response": "Release a semaphore, incrementing the internal counter by one.\n\n        When the counter is zero on entry and another thread is waiting for it\n        to become larger than zero again, wake up that thread."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreleases a semaphore, incrementing the internal counter by one. When the counter is zero on entry and another thread is waiting for it to become larger than zero again, wake up that thread. If the number of releases exceeds the number of acquires, raise a ValueError.", "response": "def release(self):\n        \"\"\"Release a semaphore, incrementing the internal counter by one.\n\n        When the counter is zero on entry and another thread is waiting for it\n        to become larger than zero again, wake up that thread.\n\n        If the number of releases exceeds the number of acquires,\n        raise a ValueError.\n\n        \"\"\"\n        with self.__cond:\n            if self.__value >= self._initial_value:\n                raise ValueError(\"Semaphore released too many times\")\n            self.__value += 1\n            self.__cond.notify()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset the internal flag to true.", "response": "def set(self):\n        \"\"\"Set the internal flag to true.\n\n        All threads waiting for the flag to become true are awakened. Threads\n        that call wait() once the flag is true will not block at all.\n\n        \"\"\"\n        with self.__cond:\n            self.__flag = True\n            self.__cond.notify_all()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nblock until the internal flag is set to True or until timeout occurs.", "response": "def wait(self, timeout=None):\n        \"\"\"Block until the internal flag is true.\n\n        If the internal flag is true on entry, return immediately. Otherwise,\n        block until another thread calls set() to set the flag to true, or until\n        the optional timeout occurs.\n\n        When the timeout argument is present and not None, it should be a\n        floating point number specifying a timeout for the operation in seconds\n        (or fractions thereof).\n\n        This method returns the internal flag on exit, so it will always return\n        True except if a timeout is given and the operation times out.\n\n        \"\"\"\n        with self.__cond:\n            if not self.__flag:\n                self.__cond.wait(timeout)\n            return self.__flag"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef start(self):\n        if not self.__initialized:\n            raise RuntimeError(\"thread.__init__() not called\")\n        if self.__started.is_set():\n            raise RuntimeError(\"threads can only be started once\")\n        if __debug__:\n            self._note(\"%s.start(): starting thread\", self)\n        with _active_limbo_lock:\n            _limbo[self] = self\n        try:\n            _start_new_thread(self.__bootstrap, ())\n        except Exception:\n            with _active_limbo_lock:\n                del _limbo[self]\n            raise\n        self.__started.wait()", "response": "Start the thread s activity."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef run(self):\n        try:\n            if self.__target:\n                self.__target(*self.__args, **self.__kwargs)\n        finally:\n            # Avoid a refcycle if the thread is running a function with\n            # an argument that has a member that points to the thread.\n            del self.__target, self.__args, self.__kwargs", "response": "Method representing the thread s activity."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef join(self, timeout=None):\n        if not self.__initialized:\n            raise RuntimeError(\"Thread.__init__() not called\")\n        if not self.__started.is_set():\n            raise RuntimeError(\"cannot join thread before it is started\")\n        if self is current_thread():\n            raise RuntimeError(\"cannot join current thread\")\n\n        if __debug__:\n            if not self.__stopped:\n                self._note(\"%s.join(): waiting until thread stops\", self)\n        self.__block.acquire()\n        try:\n            if timeout is None:\n                while not self.__stopped:\n                    self.__block.wait()\n                if __debug__:\n                    self._note(\"%s.join(): thread stopped\", self)\n            else:\n                deadline = _time() + timeout\n                while not self.__stopped:\n                    delay = deadline - _time()\n                    if delay <= 0:\n                        if __debug__:\n                            self._note(\"%s.join(): timed out\", self)\n                        break\n                    self.__block.wait(delay)\n                else:\n                    if __debug__:\n                        self._note(\"%s.join(): thread stopped\", self)\n        finally:\n            self.__block.release()", "response": "Wait until the thread terminates."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndebugs helper to print the ABC registry.", "response": "def _dump_registry(cls, file=None):\n        \"\"\"Debug helper to print the ABC registry.\"\"\"\n        print >> file, \"Class: %s.%s\" % (cls.__module__, cls.__name__)\n        print >> file, \"Inv.counter: %s\" % ABCMeta._abc_invalidation_counter\n        for name in sorted(cls.__dict__.keys()):\n            if name.startswith(\"_abc_\"):\n                value = getattr(cls, name)\n                print >> file, \"%s: %r\" % (name, value)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef b2a_qp(data, quotetabs=False, istext=True, header=False):\n    MAXLINESIZE = 76\n\n    # See if this string is using CRLF line ends\n    lf = data.find('\\n')\n    crlf = lf > 0 and data[lf-1] == '\\r'\n\n    inp = 0\n    linelen = 0\n    odata = []\n    while inp < len(data):\n        c = data[inp]\n        if (c > '~' or\n            c == '=' or\n            (header and c == '_') or\n            (c == '.' and linelen == 0 and (inp+1 == len(data) or\n                                            data[inp+1] == '\\n' or\n                                            data[inp+1] == '\\r')) or\n            (not istext and (c == '\\r' or c == '\\n')) or\n            ((c == '\\t' or c == ' ') and (inp + 1 == len(data))) or\n            (c <= ' ' and c != '\\r' and c != '\\n' and\n             (quotetabs or (not quotetabs and (c != '\\t' and c != ' '))))):\n            linelen += 3\n            if linelen >= MAXLINESIZE:\n                odata.append('=')\n                if crlf: odata.append('\\r')\n                odata.append('\\n')\n                linelen = 3\n            odata.append('=' + two_hex_digits(ord(c)))\n            inp += 1\n        else:\n            if (istext and\n                (c == '\\n' or (inp+1 < len(data) and c == '\\r' and\n                               data[inp+1] == '\\n'))):\n                linelen = 0\n                # Protect against whitespace on end of line\n                if (len(odata) > 0 and\n                    (odata[-1] == ' ' or odata[-1] == '\\t')):\n                    ch = ord(odata[-1])\n                    odata[-1] = '='\n                    odata.append(two_hex_digits(ch))\n\n                if crlf: odata.append('\\r')\n                odata.append('\\n')\n                if c == '\\r':\n                    inp += 2\n                else:\n                    inp += 1\n            else:\n                if (inp + 1 < len(data) and\n                    data[inp+1] != '\\n' and\n                    (linelen + 1) >= MAXLINESIZE):\n                    odata.append('=')\n                    if crlf: odata.append('\\r')\n                    odata.append('\\n')\n                    linelen = 0\n\n                linelen += 1\n                if header and c == ' ':\n                    c = '_'\n                odata.append(c)\n                inp += 1\n    return ''.join(odata)", "response": "Convert a string of ISO - 8601 to a QP string."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef rlecode_hqx(s):\n    if not s:\n        return ''\n    result = []\n    prev = s[0]\n    count = 1\n    # Add a dummy character to get the loop to go one extra round.\n    # The dummy must be different from the last character of s.\n    # In the same step we remove the first character, which has\n    # already been stored in prev.\n    if s[-1] == '!':\n        s = s[1:] + '?'\n    else:\n        s = s[1:] + '!'\n\n    for c in s:\n        if c == prev and count < 255:\n            count += 1\n        else:\n            if count == 1:\n                if prev != '\\x90':\n                    result.append(prev)\n                else:\n                    result += ['\\x90', '\\x00']\n            elif count < 4:\n                if prev != '\\x90':\n                    result += [prev] * count\n                else:\n                    result += ['\\x90', '\\x00'] * count\n            else:\n                if prev != '\\x90':\n                    result += [prev, '\\x90', chr(count)]\n                else:\n                    result += ['\\x90', '\\x00', '\\x90', chr(count)]\n            count = 1\n            prev = c\n\n    return ''.join(result)", "response": "This function is used to decode a string in CPython s format."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _match_abbrev(s, wordmap):\n    # Is there an exact match?\n    if s in wordmap:\n        return s\n    else:\n        # Isolate all words with s as a prefix.\n        possibilities = [word for word in wordmap.keys()\n                         if word.startswith(s)]\n        # No exact match, so there had better be just one possibility.\n        if len(possibilities) == 1:\n            return possibilities[0]\n        elif not possibilities:\n            raise BadOptionError(s)\n        else:\n            # More than one possible completion: ambiguous prefix.\n            possibilities.sort()\n            raise AmbiguousOptionError(s, possibilities)", "response": "Match an option abbreviation."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _format_text(self, text):\n        text_width = max(self.width - self.current_indent, 11)\n        indent = \" \"*self.current_indent\n        return textwrap.fill(text,\n                             text_width,\n                             initial_indent=indent,\n                             subsequent_indent=indent)", "response": "Format a text block of free - form text for inclusion in the current indentation level."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a comma - separated list of option strings & metavariables.", "response": "def format_option_strings(self, option):\n        \"\"\"Return a comma-separated list of option strings & metavariables.\"\"\"\n        if option.takes_value():\n            metavar = option.metavar or option.dest.upper()\n            short_opts = [self._short_opt_fmt % (sopt, metavar)\n                          for sopt in option._short_opts]\n            long_opts = [self._long_opt_fmt % (lopt, metavar)\n                         for lopt in option._long_opts]\n        else:\n            short_opts = option._short_opts\n            long_opts = option._long_opts\n\n        if self.short_first:\n            opts = short_opts + long_opts\n        else:\n            opts = long_opts + short_opts\n\n        return \", \".join(opts)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nupdating the option values from an arbitrary dictionary but only use keys from dict that already have a corresponding attribute COOKIEID in self.", "response": "def _update_careful(self, dict):\n        \"\"\"\n        Update the option values from an arbitrary dictionary, but only\n        use keys from dict that already have a corresponding attribute\n        in self.  Any keys in dict without a corresponding attribute\n        are silently ignored.\n        \"\"\"\n        for attr in dir(self):\n            if attr in dict:\n                dval = dict[attr]\n                if dval is not None:\n                    setattr(self, attr, dval)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef destroy(self):\n        OptionContainer.destroy(self)\n        for group in self.option_groups:\n            group.destroy()\n        del self.option_list\n        del self.option_groups\n        del self.formatter", "response": "Destroys the option container."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_args(self, args=None, values=None):\n        rargs = self._get_args(args)\n        if values is None:\n            values = self.get_default_values()\n\n        # Store the halves of the argument list as attributes for the\n        # convenience of callbacks:\n        #   rargs\n        #     the rest of the command-line (the \"r\" stands for\n        #     \"remaining\" or \"right-hand\")\n        #   largs\n        #     the leftover arguments -- ie. what's left after removing\n        #     options and their arguments (the \"l\" stands for \"leftover\"\n        #     or \"left-hand\")\n        self.rargs = rargs\n        self.largs = largs = []\n        self.values = values\n\n        try:\n            stop = self._process_args(largs, rargs, values)\n        except (BadOptionError, OptionValueError), err:\n            self.error(str(err))\n\n        args = largs + rargs\n        return self.check_values(values, args)", "response": "Parse the command - line options and return a list of values and the list of arguments."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef print_help(self, file=None):\n        if file is None:\n            file = sys.stdout\n        encoding = self._get_encoding(file)\n        # file.write(self.format_help().encode(encoding, \"replace\"))\n        file.write(self.format_help())", "response": "Print an extended help message listing all options and any\n        help text provided with them to file."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninserts item x into list a and keep it sorted assuming a is sorted.", "response": "def insort_right(a, x, lo=0, hi=None):\n    \"\"\"Insert item x in list a, and keep it sorted assuming a is sorted.\n\n    If x is already in a, insert it to the right of the rightmost x.\n\n    Optional args lo (default 0) and hi (default len(a)) bound the\n    slice of a to be searched.\n    \"\"\"\n\n    if lo < 0:\n        raise ValueError('lo must be non-negative')\n    if hi is None:\n        hi = len(a)\n    while lo < hi:\n        mid = (lo+hi)//2\n        if x < a[mid]: hi = mid\n        else: lo = mid+1\n    a.insert(lo, x)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the index where to insert item x in list a assuming a is sorted.", "response": "def bisect_right(a, x, lo=0, hi=None):\n    \"\"\"Return the index where to insert item x in list a, assuming a is sorted.\n\n    The return value i is such that all e in a[:i] have e <= x, and all e in\n    a[i:] have e > x.  So if x already appears in the list, a.insert(x) will\n    insert just after the rightmost x already there.\n\n    Optional args lo (default 0) and hi (default len(a)) bound the\n    slice of a to be searched.\n    \"\"\"\n\n    if lo < 0:\n        raise ValueError('lo must be non-negative')\n    if hi is None:\n        hi = len(a)\n    while lo < hi:\n        mid = (lo+hi)//2\n        if x < a[mid]: hi = mid\n        else: lo = mid+1\n    return lo"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the index where to insert item x in list a assuming a is sorted.", "response": "def bisect_left(a, x, lo=0, hi=None):\n    \"\"\"Return the index where to insert item x in list a, assuming a is sorted.\n\n    The return value i is such that all e in a[:i] have e < x, and all e in\n    a[i:] have e >= x.  So if x already appears in the list, a.insert(x) will\n    insert just before the leftmost x already there.\n\n    Optional args lo (default 0) and hi (default len(a)) bound the\n    slice of a to be searched.\n    \"\"\"\n\n    if lo < 0:\n        raise ValueError('lo must be non-negative')\n    if hi is None:\n        hi = len(a)\n    while lo < hi:\n        mid = (lo+hi)//2\n        if a[mid] < x: lo = mid+1\n        else: hi = mid\n    return lo"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nlock a mutex call the function with supplied argument when it is acquired.", "response": "def lock(self, function, argument):\n        \"\"\"Lock a mutex, call the function with supplied argument\n        when it is acquired.  If the mutex is already locked, place\n        function and argument in the queue.\"\"\"\n        if self.testandset():\n            function(argument)\n        else:\n            self.queue.append((function, argument))"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nunlock a mutex. If the queue is not empty, call the next function with its argument.", "response": "def unlock(self):\n        \"\"\"Unlock a mutex.  If the queue is not empty, call the next\n        function with its argument.\"\"\"\n        if self.queue:\n            function, argument = self.queue.popleft()\n            function(argument)\n        else:\n            self.locked = False"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef clear(self):\n        'od.clear() -> None.  Remove all items from od.'\n        root = self.__root\n        root[:] = [root, root, None]\n        self.__map.clear()\n        dict.clear(self)", "response": "Remove all items from od."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef popitem(self, last=True):\n        '''od.popitem() -> (k, v), return and remove a (key, value) pair.\n        Pairs are returned in LIFO order if last is true or FIFO order if false.\n\n        '''\n        if not self:\n            raise KeyError('dictionary is empty')\n        key = next(reversed(self) if last else iter(self))\n        value = self.pop(key)\n        return key, value", "response": "od. popitem - Remove and return a key value pair. last - boolean indicates whether the key is last or not."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef update(*args, **kwds):\n        '''Like dict.update() but add counts instead of replacing them.\n\n        Source can be an iterable, a dictionary, or another Counter instance.\n\n        >>> c = Counter('which')\n        >>> c.update('witch')           # add elements from another iterable\n        >>> d = Counter('watch')\n        >>> c.update(d)                 # add elements from another counter\n        >>> c['h']                      # four 'h' in which, witch, and watch\n        4\n\n        '''\n        # The regular dict.update() operation makes no sense here because the\n        # replace behavior results in the some of original untouched counts\n        # being mixed-in with all of the other counts for a mismash that\n        # doesn't have a straight-forward interpretation in most counting\n        # contexts.  Instead, we implement straight-addition.  Both the inputs\n        # and outputs are allowed to contain zero and negative counts.\n\n        if not args:\n            raise TypeError(\"descriptor 'update' of 'Counter' object \"\n                            \"needs an argument\")\n        self = args[0]\n        args = args[1:]\n        if len(args) > 1:\n            raise TypeError('expected at most 1 arguments, got %d' % len(args))\n        iterable = args[0] if args else None\n        if iterable is not None:\n            if isinstance(iterable, Mapping):\n                if self:\n                    self_get = self.get\n                    for elem, count in iterable.iteritems():\n                        self[elem] = self_get(elem, 0) + count\n                else:\n                    super(Counter, self).update(iterable) # fast path when counter is empty\n            else:\n                self_get = self.get\n                for elem in iterable:\n                    self[elem] = self_get(elem, 0) + 1\n        if kwds:\n            self.update(kwds)", "response": "Like dict. update but add counts instead of replacing them."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef subtract(*args, **kwds):\n        '''Like dict.update() but subtracts counts instead of replacing them.\n        Counts can be reduced below zero.  Both the inputs and outputs are\n        allowed to contain zero and negative counts.\n\n        Source can be an iterable, a dictionary, or another Counter instance.\n\n        >>> c = Counter('which')\n        >>> c.subtract('witch')             # subtract elements from another iterable\n        >>> c.subtract(Counter('watch'))    # subtract elements from another counter\n        >>> c['h']                          # 2 in which, minus 1 in witch, minus 1 in watch\n        0\n        >>> c['w']                          # 1 in which, minus 1 in witch, minus 1 in watch\n        -1\n\n        '''\n        if not args:\n            raise TypeError(\"descriptor 'subtract' of 'Counter' object \"\n                            \"needs an argument\")\n        self = args[0]\n        args = args[1:]\n        if len(args) > 1:\n            raise TypeError('expected at most 1 arguments, got %d' % len(args))\n        iterable = args[0] if args else None\n        if iterable is not None:\n            self_get = self.get\n            if isinstance(iterable, Mapping):\n                for elem, count in iterable.items():\n                    self[elem] = self_get(elem, 0) - count\n            else:\n                for elem in iterable:\n                    self[elem] = self_get(elem, 0) - 1\n        if kwds:\n            self.subtract(kwds)", "response": "Like dict. update but subtracts counts instead of replacing them."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef digest(self):\n\n    A = self.A\n    B = self.B\n    C = self.C\n    D = self.D\n    input = [] + self.input\n    count = [] + self.count\n\n    index = (self.count[0] >> 3) & 0x3f\n\n    if index < 56:\n      padLen = 56 - index\n    else:\n      padLen = 120 - index\n\n    padding = [b'\\200'] + [b'\\000'] * 63\n    self.update(padding[:padLen])\n\n    # Append length (before padding).\n    bits = _bytelist2long(self.input[:56]) + count\n\n    self._transform(bits)\n\n    # Store state in digest.\n    digest = struct.pack(\"<IIII\", self.A, self.B, self.C, self.D)\n\n    self.A = A\n    self.B = B\n    self.C = C\n    self.D = D\n    self.input = input\n    self.count = count\n\n    return digest", "response": "Terminate the message - digest computation and return digest."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a copy of the md5 object.", "response": "def copy(self):\n    \"\"\"Return a clone object.\n\n    Return a copy ('clone') of the md5 object. This can be used\n    to efficiently compute the digests of strings that share\n    a common initial substring.\n    \"\"\"\n    if 0:  # set this to 1 to make the flow space crash\n      return copy.deepcopy(self)\n    clone = self.__class__()\n    clone.length = self.length\n    clone.count = [] + self.count[:]\n    clone.input = [] + self.input\n    clone.A = self.A\n    clone.B = self.B\n    clone.C = self.C\n    clone.D = self.D\n    return clone"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef compile(pattern, flags, code, groups=0, groupindex={}, indexgroup=[None]):\n    return SRE_Pattern(pattern, flags, code, groups, groupindex, indexgroup)", "response": "Compiles a pattern descriptor to a SRE_Pattern object."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef search(self, string, pos=0, endpos=sys.maxint):\n        state = _State(string, pos, endpos, self.flags)\n        if state.search(self._code):\n            return SRE_Match(self, state)\n        else:\n            return None", "response": "Search through string looking for a location where this regular\n        expression produces a match and return a corresponding MatchObject\n        instance. Return None if no match is found. Return None if no match is found."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a list of all non - overlapping matches of pattern in string.", "response": "def findall(self, string, pos=0, endpos=sys.maxint):\n        \"\"\"Return a list of all non-overlapping matches of pattern in string.\"\"\"\n        matchlist = []\n        state = _State(string, pos, endpos, self.flags)\n        while state.start <= state.end:\n            state.reset()\n            state.string_position = state.start\n            if not state.search(self._code):\n                break\n            match = SRE_Match(self, state)\n            if self.groups == 0 or self.groups == 1:\n                item = match.group(self.groups)\n            else:\n                item = match.groups(\"\")\n            matchlist.append(item)\n            if state.string_position == state.start:\n                state.start += 1\n            else:\n                state.start = state.string_position\n        return matchlist"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the string obtained by replacing the leftmost non - overlapping occurrences of pattern in string by the replacement repl.", "response": "def sub(self, repl, string, count=0):\n        \"\"\"Return the string obtained by replacing the leftmost non-overlapping\n        occurrences of pattern in string by the replacement repl.\"\"\"\n        return self._subx(repl, string, count, False)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the tuple ( new_string number_of_subs_made ) found by replacing repl.", "response": "def subn(self, repl, string, count=0):\n        \"\"\"Return the tuple (new_string, number_of_subs_made) found by replacing\n        the leftmost non-overlapping occurrences of pattern with the replacement\n        repl.\"\"\"\n        return self._subx(repl, string, count, True)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef split(self, string, maxsplit=0):\n        splitlist = []\n        state = _State(string, 0, sys.maxint, self.flags)\n        n = 0\n        last = state.start\n        while not maxsplit or n < maxsplit:\n            state.reset()\n            state.string_position = state.start\n            if not state.search(self._code):\n                break\n            if state.start == state.string_position: # zero-width match\n                if last == state.end:                # or end of string\n                    break\n                state.start += 1\n                continue\n            splitlist.append(string[last:state.start])\n            # add groups (if any)\n            if self.groups:\n                match = SRE_Match(self, state)\n                # TODO: Use .extend once it is implemented.\n                # splitlist.extend(list(match.groups(None)))\n                splitlist += (list(match.groups(None)))\n            n += 1\n            last = state.start = state.string_position\n        splitlist.append(string[last:state.end])\n        return splitlist", "response": "Split string by the occurrences of pattern."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a list of all non - overlapping matches of pattern in string.", "response": "def finditer(self, string, pos=0, endpos=sys.maxint):\n        \"\"\"Return a list of all non-overlapping matches of pattern in string.\"\"\"\n        scanner = self.scanner(string, pos, endpos)\n        return iter(scanner.search, None)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _create_regs(self, state):\n        regs = [(state.start, state.string_position)]\n        for group in range(self.re.groups):\n            mark_index = 2 * group\n            if mark_index + 1 < len(state.marks) \\\n                                    and state.marks[mark_index] is not None \\\n                                    and state.marks[mark_index + 1] is not None:\n                regs.append((state.marks[mark_index], state.marks[mark_index + 1]))\n            else:\n                regs.append((-1, -1))\n        return tuple(regs)", "response": "Creates a tuple of index pairs representing matched groups."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a tuple containing all the subgroups of the match. The default argument is used for groups that did not participate in the the match.", "response": "def groups(self, default=None):\n        \"\"\"Returns a tuple containing all the subgroups of the match. The\n        default argument is used for groups that did not participate in the\n        match (defaults to None).\"\"\"\n        groups = []\n        for indices in self.regs[1:]:\n            if indices[0] >= 0:\n                groups.append(self.string[indices[0]:indices[1]])\n            else:\n                groups.append(default)\n        return tuple(groups)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a dictionary containing all the named subgroups of the match.", "response": "def groupdict(self, default=None):\n        \"\"\"Return a dictionary containing all the named subgroups of the match.\n        The default argument is used for groups that did not participate in the\n        match (defaults to None).\"\"\"\n        groupdict = {}\n        for key, value in self.re.groupindex.items():\n            groupdict[key] = self._get_slice(value, default)\n        return groupdict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning one or more subgroups of the match. Each argument is either a group index or a group name.", "response": "def group(self, *args):\n        \"\"\"Returns one or more subgroups of the match. Each argument is either a\n        group index or a group name.\"\"\"\n        if len(args) == 0:\n            args = (0,)\n        grouplist = []\n        for group in args:\n            grouplist.append(self._get_slice(self._get_index(group), None))\n        if len(grouplist) == 1:\n            return grouplist[0]\n        else:\n            return tuple(grouplist)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef fast_search(self, pattern_codes):\n        # pattern starts with a known prefix\n        # <5=length> <6=skip> <7=prefix data> <overlap data>\n        flags = pattern_codes[2]\n        prefix_len = pattern_codes[5]\n        prefix_skip = pattern_codes[6] # don't really know what this is good for\n        prefix = pattern_codes[7:7 + prefix_len]\n        overlap = pattern_codes[7 + prefix_len - 1:pattern_codes[1] + 1]\n        pattern_codes = pattern_codes[pattern_codes[1] + 1:]\n        i = 0\n        string_position = self.string_position\n        while string_position < self.end:\n            while True:\n                if ord(self.string[string_position]) != prefix[i]:\n                    if i == 0:\n                        break\n                    else:\n                        i = overlap[i]\n                else:\n                    i += 1\n                    if i == prefix_len:\n                        # found a potential match\n                        self.start = string_position + 1 - prefix_len\n                        self.string_position = string_position + 1 \\\n                                                     - prefix_len + prefix_skip\n                        if flags & SRE_INFO_LITERAL:\n                            return True # matched all of pure literal pattern\n                        if self.match(pattern_codes[2 * prefix_skip:]):\n                            return True\n                        i = overlap[i]\n                    break\n            string_position += 1\n        return False", "response": "Searches for a pattern in a string as fast as possible using information from\n        an optimization info block."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef push_new_context(self, pattern_offset):\n        child_context = _MatchContext(self.state,\n            self.pattern_codes[self.code_position + pattern_offset:])\n        self.state.context_stack.append(child_context)\n        return child_context", "response": "Creates a new child context and pushes it on the stack."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn True if the current context matches False if it doesn t and None if it doesn t match.", "response": "def match(self, context):\n        \"\"\"Returns True if the current context matches, False if it doesn't and\n        None if matching is not finished, ie must be resumed after child\n        contexts have been matched.\"\"\"\n        while context.remaining_codes() > 0 and context.has_matched is None:\n            opcode = context.peek_code()\n            if not self.dispatch(opcode, context):\n                return None\n        if context.has_matched is None:\n            context.has_matched = False\n        return context.has_matched"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef dispatch(self, opcode, context):\n        if id(context) in self.executing_contexts:\n            generator = self.executing_contexts[id(context)]\n            del self.executing_contexts[id(context)]\n            has_finished = generator.next()\n        else:\n            method = self.DISPATCH_TABLE.get(opcode, _OpcodeDispatcher.unknown)\n            has_finished = method(self, context)\n            if hasattr(has_finished, \"next\"): # avoid using the types module\n                generator = has_finished\n                has_finished = generator.next()\n        if not has_finished:\n            self.executing_contexts[id(context)] = generator\n        return has_finished", "response": "Dispatches a given opcode on a given context. Returns True if the context\n            is done matching False otherwise."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef check_charset(self, ctx, char):\n        self.set_dispatcher.reset(char)\n        save_position = ctx.code_position\n        result = None\n        while result is None:\n            result = self.set_dispatcher.dispatch(ctx.peek_code(), ctx)\n        ctx.code_position = save_position\n        return result", "response": "Checks whether a character matches the set of arbitrary length."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef count_repetitions(self, ctx, maxcount):\n        count = 0\n        real_maxcount = ctx.state.end - ctx.string_position\n        if maxcount < real_maxcount and maxcount != MAXREPEAT:\n            real_maxcount = maxcount\n        # XXX could special case every single character pattern here, as in C.\n        # This is a general solution, a bit hackisch, but works and should be\n        # efficient.\n        code_position = ctx.code_position\n        string_position = ctx.string_position\n        ctx.skip_code(4)\n        reset_position = ctx.code_position\n        while count < real_maxcount:\n            # this works because the single character pattern is followed by\n            # a success opcode\n            ctx.code_position = reset_position\n            self.dispatch(ctx.peek_code(), ctx)\n            if ctx.has_matched is False: # could be None as well\n                break\n            count += 1\n        ctx.has_matched = None\n        ctx.code_position = code_position\n        ctx.string_position = string_position\n        return count", "response": "Returns the number of repetitions of a single item starting from the current string position."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef extract(s):\n    res = decoder.match(s)\n    if res is None: raise NotANumber, s\n    sign, intpart, fraction, exppart = res.group(1,2,3,4)\n    if sign == '+': sign = ''\n    if fraction: fraction = fraction[1:]\n    if exppart: expo = int(exppart[1:])\n    else: expo = 0\n    return sign, intpart, fraction, expo", "response": "Extracts a non - zero or non - negative integer from a non - zero or non - negative integer."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef unexpo(intpart, fraction, expo):\n    if expo > 0: # Move the point left\n        f = len(fraction)\n        intpart, fraction = intpart + fraction[:expo], fraction[expo:]\n        if expo > f:\n            intpart = intpart + '0'*(expo-f)\n    elif expo < 0: # Move the point right\n        i = len(intpart)\n        intpart, fraction = intpart[:expo], intpart[expo:] + fraction\n        if expo < -i:\n            fraction = '0'*(-expo-i) + fraction\n    return intpart, fraction", "response": "Remove the exponent by changing intpart and fraction."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrounding or extend the fraction to size digs.", "response": "def roundfrac(intpart, fraction, digs):\n    \"\"\"Round or extend the fraction to size digs.\"\"\"\n    f = len(fraction)\n    if f <= digs:\n        return intpart, fraction + '0'*(digs-f)\n    i = len(intpart)\n    if i+digs < 0:\n        return '0'*-digs, ''\n    total = intpart + fraction\n    nextdigit = total[i+digs]\n    if nextdigit >= '5': # Hard case: increment last digit, may have carry!\n        n = i + digs - 1\n        while n >= 0:\n            if total[n] != '9': break\n            n = n-1\n        else:\n            total = '0' + total\n            i = i+1\n            n = 0\n        total = total[:n] + chr(ord(total[n]) + 1) + '0'*(len(total)-n-1)\n        intpart, fraction = total[:i], total[i:]\n    if digs >= 0:\n        return intpart, fraction[:digs]\n    else:\n        return intpart[:digs] + '0'*-digs, ''"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nformats x as - ddd. ddd with digs digits after the point and at least one digit before the point .", "response": "def fix(x, digs):\n    \"\"\"Format x as [-]ddd.ddd with 'digs' digits after the point\n    and at least one digit before.\n    If digs <= 0, the point is suppressed.\"\"\"\n    if type(x) != type(''): x = repr(x)\n    try:\n        sign, intpart, fraction, expo = extract(x)\n    except NotANumber:\n        return x\n    intpart, fraction = unexpo(intpart, fraction, expo)\n    intpart, fraction = roundfrac(intpart, fraction, digs)\n    while intpart and intpart[0] == '0': intpart = intpart[1:]\n    if intpart == '': intpart = '0'\n    if digs > 0: return sign + intpart + '.' + fraction\n    else: return sign + intpart"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nformat x as - d. dddE [+- ] ddd with digs digits after the point and exactly one digit before the point .", "response": "def sci(x, digs):\n    \"\"\"Format x as [-]d.dddE[+-]ddd with 'digs' digits after the point\n    and exactly one digit before.\n    If digs is <= 0, one digit is kept and the point is suppressed.\"\"\"\n    if type(x) != type(''): x = repr(x)\n    sign, intpart, fraction, expo = extract(x)\n    if not intpart:\n        while fraction and fraction[0] == '0':\n            fraction = fraction[1:]\n            expo = expo - 1\n        if fraction:\n            intpart, fraction = fraction[0], fraction[1:]\n            expo = expo - 1\n        else:\n            intpart = '0'\n    else:\n        expo = expo + len(intpart) - 1\n        intpart, fraction = intpart[0], intpart[1:] + fraction\n    digs = max(0, digs)\n    intpart, fraction = roundfrac(intpart, fraction, digs)\n    if len(intpart) > 1:\n        intpart, fraction, expo = \\\n            intpart[0], intpart[1:] + fraction[:-1], \\\n            expo + len(intpart) - 1\n    s = sign + intpart\n    if digs > 0: s = s + '.' + fraction\n    e = repr(abs(expo))\n    e = '0'*(3-len(e)) + e\n    if expo < 0: e = '-' + e\n    else: e = '+' + e\n    return s + 'e' + e"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getrandbits(self, k):\n    if k <= 0:\n      raise ValueError('number of bits must be greater than zero')\n    if k != int(k):\n      raise TypeError('number of bits should be an integer')\n    numbytes = (k + 7) // 8                       # bits / 8 and rounded up\n    x = _int_from_bytes(_gorandom(numbytes))\n    return x >> (numbytes * 8 - k)", "response": "Generates an int with k random bits."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _randbelow(self, n):\n    # TODO\n    # change once int.bit_length is implemented.\n    # k = n.bit_length()\n    k = _int_bit_length(n)\n    r = self.getrandbits(k)\n    while r >= n:\n      r = self.getrandbits(k)\n    return r", "response": "Return a random int in the range [ 0 n )."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getopt(args, shortopts, longopts = []):\n\n    opts = []\n    if type(longopts) == type(\"\"):\n        longopts = [longopts]\n    else:\n        longopts = list(longopts)\n    while args and args[0].startswith('-') and args[0] != '-':\n        if args[0] == '--':\n            args = args[1:]\n            break\n        if args[0].startswith('--'):\n            opts, args = do_longs(opts, args[0][2:], longopts, args[1:])\n        else:\n            opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:])\n\n    return opts, args", "response": "Parses command line options and parameter list."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef gnu_getopt(args, shortopts, longopts = []):\n\n    opts = []\n    prog_args = []\n    if isinstance(longopts, str):\n        longopts = [longopts]\n    else:\n        longopts = list(longopts)\n\n    # Allow options after non-option arguments?\n    if shortopts.startswith('+'):\n        shortopts = shortopts[1:]\n        all_options_first = True\n    elif os.environ.get(\"POSIXLY_CORRECT\"):\n        all_options_first = True\n    else:\n        all_options_first = False\n\n    while args:\n        if args[0] == '--':\n            prog_args += args[1:]\n            break\n\n        if args[0][:2] == '--':\n            opts, args = do_longs(opts, args[0][2:], longopts, args[1:])\n        elif args[0][:1] == '-' and args[0] != '-':\n            opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:])\n        else:\n            if all_options_first:\n                prog_args += args\n                break\n            else:\n                prog_args.append(args[0])\n                args = args[1:]\n\n    return opts, prog_args", "response": "This function is used by GNU to parse GNU command line options."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef filter(names, pat):\n    import os\n    # import posixpath\n    result=[]\n    # pat=os.path.normcase(pat)\n    try:\n        re_pat = _cache[pat]\n    except KeyError:\n        res = translate(pat)\n        if len(_cache) >= _MAXCACHE:\n            # _cache.clear()\n            globals()['_cache'] = {}\n        _cache[pat] = re_pat = re.compile(res)\n    match = re_pat.match\n    # if os.path is posixpath:\n    if 1:\n        # normcase on posix is NOP. Optimize it away from the loop.\n        for name in names:\n            if match(name):\n                result.append(name)\n    else:\n        for name in names:\n            if match(os.path.normcase(name)):\n                result.append(name)\n    return result", "response": "Return the subset of the list NAMES that match PAT"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef fnmatchcase(name, pat):\n\n    try:\n        re_pat = _cache[pat]\n    except KeyError:\n        res = translate(pat)\n        if len(_cache) >= _MAXCACHE:\n            # _cache.clear()\n            globals()['_cache'] = {}\n        _cache[pat] = re_pat = re.compile(res)\n    return re_pat.match(name) is not None", "response": "Test whether FILENAME matches PATTERN including case."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef translate(pat):\n\n    i, n = 0, len(pat)\n    res = ''\n    while i < n:\n        c = pat[i]\n        i = i+1\n        if c == '*':\n            res = res + '.*'\n        elif c == '?':\n            res = res + '.'\n        elif c == '[':\n            j = i\n            if j < n and pat[j] == '!':\n                j = j+1\n            if j < n and pat[j] == ']':\n                j = j+1\n            while j < n and pat[j] != ']':\n                j = j+1\n            if j >= n:\n                res = res + '\\\\['\n            else:\n                stuff = pat[i:j].replace('\\\\','\\\\\\\\')\n                i = j+1\n                if stuff[0] == '!':\n                    stuff = '^' + stuff[1:]\n                elif stuff[0] == '^':\n                    stuff = '\\\\' + stuff\n                res = '%s[%s]' % (res, stuff)\n        else:\n            res = res + re.escape(c)\n    return res + '\\Z(?ms)'", "response": "Translate a shell PATTERN to a regular expression."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef task_done(self):\n        self.all_tasks_done.acquire()\n        try:\n            unfinished = self.unfinished_tasks - 1\n            if unfinished <= 0:\n                if unfinished < 0:\n                    raise ValueError('task_done() called too many times')\n                self.all_tasks_done.notify_all()\n            self.unfinished_tasks = unfinished\n        finally:\n            self.all_tasks_done.release()", "response": "Indicate that a task is complete."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the approximate size of the queue ( not reliable.", "response": "def qsize(self):\n        \"\"\"Return the approximate size of the queue (not reliable!).\"\"\"\n        self.mutex.acquire()\n        n = self._qsize()\n        self.mutex.release()\n        return n"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns True if the queue is empty False otherwise.", "response": "def empty(self):\n        \"\"\"Return True if the queue is empty, False otherwise (not reliable!).\"\"\"\n        self.mutex.acquire()\n        n = not self._qsize()\n        self.mutex.release()\n        return n"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef full(self):\n        self.mutex.acquire()\n        n = 0 < self.maxsize == self._qsize()\n        self.mutex.release()\n        return n", "response": "Return True if the queue is full False otherwise."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nputting an item into the queue.", "response": "def put(self, item, block=True, timeout=None):\n        \"\"\"Put an item into the queue.\n\n        If optional args 'block' is true and 'timeout' is None (the default),\n        block if necessary until a free slot is available. If 'timeout' is\n        a non-negative number, it blocks at most 'timeout' seconds and raises\n        the Full exception if no free slot was available within that time.\n        Otherwise ('block' is false), put an item on the queue if a free slot\n        is immediately available, else raise the Full exception ('timeout'\n        is ignored in that case).\n        \"\"\"\n        self.not_full.acquire()\n        try:\n            if self.maxsize > 0:\n                if not block:\n                    if self._qsize() == self.maxsize:\n                        raise Full\n                elif timeout is None:\n                    while self._qsize() == self.maxsize:\n                        self.not_full.wait()\n                elif timeout < 0:\n                    raise ValueError(\"'timeout' must be a non-negative number\")\n                else:\n                    endtime = _time() + timeout\n                    while self._qsize() == self.maxsize:\n                        remaining = endtime - _time()\n                        if remaining <= 0.0:\n                            raise Full\n                        self.not_full.wait(remaining)\n            self._put(item)\n            self.unfinished_tasks += 1\n            self.not_empty.notify()\n        finally:\n            self.not_full.release()"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef calculate_transitive_deps(modname, script, gopath):\n  deps = set()\n  def calc(modname, script):\n    if modname in deps:\n      return\n    deps.add(modname)\n    for imp in collect_imports(modname, script, gopath):\n      if imp.is_native:\n        deps.add(imp.name)\n        continue\n      parts = imp.name.split('.')\n      calc(imp.name, imp.script)\n      if len(parts) == 1:\n        continue\n      # For submodules, the parent packages are also deps.\n      package_dir, filename = os.path.split(imp.script)\n      if filename == '__init__.py':\n        package_dir = os.path.dirname(package_dir)\n      for i in xrange(len(parts) - 1, 0, -1):\n        modname = '.'.join(parts[:i])\n        script = os.path.join(package_dir, '__init__.py')\n        calc(modname, script)\n        package_dir = os.path.dirname(package_dir)\n  calc(modname, script)\n  deps.remove(modname)\n  return deps", "response": "Determines all modules that script transitively depends upon."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nprocessing a future import statement returning a set of flags it defines.", "response": "def _make_future_features(node):\n  \"\"\"Processes a future import statement, returning set of flags it defines.\"\"\"\n  assert isinstance(node, ast.ImportFrom)\n  assert node.module == '__future__'\n  features = FutureFeatures()\n  for alias in node.names:\n    name = alias.name\n    if name in _FUTURE_FEATURES:\n      if name not in _IMPLEMENTED_FUTURE_FEATURES:\n        msg = 'future feature {} not yet implemented by grumpy'.format(name)\n        raise util.ParseError(node, msg)\n      setattr(features, name, True)\n    elif name == 'braces':\n      raise util.ParseError(node, 'not a chance')\n    elif name not in _REDUNDANT_FUTURE_FEATURES:\n      msg = 'future feature {} is not defined'.format(name)\n      raise util.ParseError(node, msg)\n  return features"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\naccumulate a set of flags for the compiler __future__ imports.", "response": "def parse_future_features(mod):\n  \"\"\"Accumulates a set of flags for the compiler __future__ imports.\"\"\"\n  assert isinstance(mod, ast.Module)\n  found_docstring = False\n  for node in mod.body:\n    if isinstance(node, ast.ImportFrom):\n      if node.module == '__future__':\n        return node, _make_future_features(node)\n      break\n    elif isinstance(node, ast.Expr) and not found_docstring:\n      if not isinstance(node.value, ast.Str):\n        break\n      found_docstring = True\n    else:\n      break\n  return None, FutureFeatures()"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef contextmanager(func):\n    @wraps(func)\n    def helper(*args, **kwds):\n        return GeneratorContextManager(func(*args, **kwds))\n    return helper", "response": "Decorator that returns a generator that yields all the available items in the sequence."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncombines multiple context managers into a single nested context manager.", "response": "def nested(*managers):\n    \"\"\"Combine multiple context managers into a single nested context manager.\n\n   This function has been deprecated in favour of the multiple manager form\n   of the with statement.\n\n   The one advantage of this function over the multiple manager form of the\n   with statement is that argument unpacking allows it to be\n   used with a variable number of context managers as follows:\n\n      with nested(*managers):\n          do_something()\n\n    \"\"\"\n    warn(\"With-statements now directly support multiple context managers\",\n         DeprecationWarning, 3)\n    exits = []\n    vars = []\n    exc = (None, None, None)\n    try:\n        for mgr in managers:\n            exit = mgr.__exit__\n            enter = mgr.__enter__\n            vars.append(enter())\n            exits.append(exit)\n        yield vars\n    except:\n        exc = sys.exc_info()\n    finally:\n        while exits:\n            exit = exits.pop()\n            try:\n                if exit(*exc):\n                    exc = (None, None, None)\n            except:\n                exc = sys.exc_info()\n        if exc != (None, None, None):\n            # Don't rely on sys.exc_info() still containing\n            # the right information. Another exception may\n            # have been raised and caught by an exit method\n            raise exc[0], exc[1], exc[2]"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef decode(self, s, _w=WHITESPACE.match):\n        obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n        end = _w(s, end).end()\n        if end != len(s):\n            raise ValueError(errmsg(\"Extra data\", s, end, len(s)))\n        return obj", "response": "Decode a string or unicode object into a Python object."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef tf_loss(self, states, internals, reward, update, reference=None):\n        prediction = self.predict(states=states, internals=internals, update=update)\n        return tf.nn.l2_loss(t=(prediction - reward))", "response": "Returns the TensorFlow operations for calculating the L2 loss between predicted state values and actual rewards."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the TensorFlow variables used by the baseline.", "response": "def get_variables(self, include_nontrainable=False):\n        \"\"\"\n        Returns the TensorFlow variables used by the baseline.\n\n        Returns:\n            List of variables\n        \"\"\"\n        if include_nontrainable:\n            return [self.all_variables[key] for key in sorted(self.all_variables)]\n        else:\n            return [self.variables[key] for key in sorted(self.variables)]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a baseline from a specification dict.", "response": "def from_spec(spec, kwargs=None):\n        \"\"\"\n        Creates a baseline from a specification dict.\n        \"\"\"\n        baseline = util.get_object(\n            obj=spec,\n            predefined_objects=tensorforce.core.baselines.baselines,\n            kwargs=kwargs\n        )\n        assert isinstance(baseline, Baseline)\n        return baseline"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef reset(self):\n        # Send command.\n        self.protocol.send({\"cmd\": \"reset\"}, self.socket)\n        # Wait for response.\n        response = self.protocol.recv(self.socket)\n        # Extract observations.\n        return self.extract_observation(response)", "response": "reset the current state of the current set of items"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nexecuting a single step in the UE4 game.", "response": "def execute(self, action):\n        \"\"\"\n        Executes a single step in the UE4 game. This step may be comprised of one or more actual game ticks for all of\n        which the same given\n        action- and axis-inputs (or action number in case of discretized actions) are repeated.\n        UE4 distinguishes between action-mappings, which are boolean actions (e.g. jump or dont-jump) and axis-mappings,\n        which are continuous actions\n        like MoveForward with values between -1.0 (run backwards) and 1.0 (run forwards), 0.0 would mean: stop.\n        \"\"\"\n        action_mappings, axis_mappings = [], []\n\n        # TODO: what if more than one actions are passed?\n\n        # Discretized -> each action is an int\n        if self.discretize_actions:\n            # Pull record from discretized_actions, which will look like: [A, Right, SpaceBar].\n            combination = self.discretized_actions[action]\n            # Translate to {\"axis_mappings\": [('A', 1.0), (Right, 1.0)], \"action_mappings\": [(SpaceBar, True)]}\n            for key, value in combination:\n                # Action mapping (True or False).\n                if isinstance(value, bool):\n                    action_mappings.append((key, value))\n                # Axis mapping: always use 1.0 as value as UE4 already multiplies with the correct scaling factor.\n                else:\n                    axis_mappings.append((key, value))\n        # Non-discretized: Each action is a dict of action- and axis-mappings defined in UE4 game's input settings.\n        # Re-translate Incoming action names into keyboard keys for the server.\n        elif action:\n            try:\n                action_mappings, axis_mappings = self.translate_abstract_actions_to_keys(action)\n            except KeyError as e:\n                raise TensorForceError(\"Action- or axis-mapping with name '{}' not defined in connected UE4 game!\".\n                                       format(e))\n\n        # message = {\"cmd\": \"step\", 'delta_time': 0.33,\n        #     'actions': [('X', True), ('Y', False)],\n        #     'axes': [('Left': 1.0), ('Up': -1.0)]\n        # }\n        message = dict(\n            cmd=\"step\",\n            delta_time=self.delta_time,\n            num_ticks=self.num_ticks,\n            actions=action_mappings,\n            axes=axis_mappings\n        )\n        self.protocol.send(message, self.socket)\n        # Wait for response (blocks).\n        response = self.protocol.recv(self.socket)\n        r = response.pop(b\"_reward\", 0.0)\n        is_terminal = response.pop(b\"_is_terminal\", False)\n\n        obs = self.extract_observation(response)\n        # Cache last observation\n        self.last_observation = obs\n        return obs, is_terminal, r"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef translate_abstract_actions_to_keys(self, abstract):\n\n        # Solve single tuple with name and value -> should become a list (len=1) of this tuple.\n        if len(abstract) >= 2 and not isinstance(abstract[1], (list, tuple)):\n            abstract = list((abstract,))\n\n        # Now go through the list and translate each axis into an actual keyboard key (or mouse event/etc..).\n        actions, axes = [], []\n        for a in abstract:\n            # first_key = key-name (action mapping or discretized axis mapping) OR tuple (key-name, scale) (continuous\n            # axis mapping)\n            first_key = self.action_space_desc[a[0]][\"keys\"][0]\n            # action mapping\n            if isinstance(first_key, (bytes, str)):\n                actions.append((first_key, a[1]))\n            # axis mapping\n            elif isinstance(first_key, tuple):\n                axes.append((first_key[0], a[1] * first_key[1]))\n            else:\n                raise TensorForceError(\"action_space_desc contains unsupported type for key {}!\".format(a[0]))\n\n        return actions, axes", "response": "Translate a list of tuples with some key and translated value into a list of tuples with some key and translated value."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a list of discrete action(-combinations) in case we want to learn with a discrete set of actions, but only have action-combinations (maybe even continuous) available from the env. E.g. the UE4 game has the following action/axis-mappings: ```javascript { 'Fire': {'type': 'action', 'keys': ('SpaceBar',)}, 'MoveRight': {'type': 'axis', 'keys': (('Right', 1.0), ('Left', -1.0), ('A', -1.0), ('D', 1.0))}, } ``` -> this method will discretize them into the following 6 discrete actions: ```javascript [ [(Right, 0.0),(SpaceBar, False)], [(Right, 0.0),(SpaceBar, True)] [(Right, -1.0),(SpaceBar, False)], [(Right, -1.0),(SpaceBar, True)], [(Right, 1.0),(SpaceBar, False)], [(Right, 1.0),(SpaceBar, True)], ] ```", "response": "def discretize_action_space_desc(self):\n        \"\"\"\n        Creates a list of discrete action(-combinations) in case we want to learn with a discrete set of actions,\n        but only have action-combinations (maybe even continuous) available from the env.\n        E.g. the UE4 game has the following action/axis-mappings:\n\n        ```javascript\n        {\n        'Fire':\n            {'type': 'action', 'keys': ('SpaceBar',)},\n        'MoveRight':\n            {'type': 'axis', 'keys': (('Right', 1.0), ('Left', -1.0), ('A', -1.0), ('D', 1.0))},\n        }\n        ```\n\n        -> this method will discretize them into the following 6 discrete actions:\n\n        ```javascript\n        [\n        [(Right, 0.0),(SpaceBar, False)],\n        [(Right, 0.0),(SpaceBar, True)]\n        [(Right, -1.0),(SpaceBar, False)],\n        [(Right, -1.0),(SpaceBar, True)],\n        [(Right, 1.0),(SpaceBar, False)],\n        [(Right, 1.0),(SpaceBar, True)],\n        ]\n        ```\n\n        \"\"\"\n        # Put all unique_keys lists in one list and itertools.product that list.\n        unique_list = []\n        for nice, record in self.action_space_desc.items():\n            list_for_record = []\n            if record[\"type\"] == \"axis\":\n                # The main key for this record (always the first one)\n                head_key = record[\"keys\"][0][0]\n                # The reference value (divide by this one to get the others)\n                head_value = record[\"keys\"][0][1]\n                # The zero key (idle action; axis scale=0.0)\n                list_for_record.append((head_key, 0.0))\n                set_ = set()\n                for key_and_scale in self.action_space_desc[nice][\"keys\"]:\n                    # Build unique lists of mappings (each axis value should only be represented once).\n                    if key_and_scale[1] not in set_:\n                        list_for_record.append((head_key, key_and_scale[1] / head_value))\n                        set_.add(key_and_scale[1])\n            else:\n                # Action-mapping\n                list_for_record = [(record[\"keys\"][0], False), (record[\"keys\"][0], True)]\n            unique_list.append(list_for_record)\n\n        def so(in_):\n            # in_ is List[Tuple[str,any]] -> sort by concat'd sequence of str(any's)\n            st = \"\"\n            for i in in_:\n                st += str(i[1])\n            return st\n\n        # Then sort and get the entire list of all possible sorted meaningful key-combinations.\n        combinations = list(itertools.product(*unique_list))\n        combinations = list(map(lambda x: sorted(list(x), key=lambda y: y[0]), combinations))\n        combinations = sorted(combinations, key=so)\n        # Store that list as discretized_actions.\n        self.discretized_actions = combinations"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nresetting the environment to its initial state.", "response": "def reset(self):\n        \"\"\"\n        Resets the environment to its initialization state. This method needs to be called to start a\n        new episode after the last episode ended.\n\n        :return: initial state\n        \"\"\"\n        self.level.reset()  # optional: episode=-1, seed=None\n        return self.level.observations()[self.state_attribute]"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef execute(self, action):\n        adjusted_action = list()\n        for action_spec in self.level.action_spec():\n            if action_spec['min'] == -1 and action_spec['max'] == 1:\n                adjusted_action.append(action[action_spec['name']] - 1)\n            else:\n                adjusted_action.append(action[action_spec['name']])  # clip?\n        action = np.array(adjusted_action, dtype=np.intc)\n\n        reward = self.level.step(action=action, num_steps=self.repeat_action)\n        state = self.level.observations()['RGB_INTERLACED']\n        terminal = not self.level.is_running()\n        return state, terminal, reward", "response": "Execute the action given in the action_spec and return the next state terminal and the reward."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef tf_initialize(self, x_init, b):\n        if x_init is None:\n            # Initial guess is zero vector if not given.\n            x_init = [tf.zeros(shape=util.shape(t)) for t in b]\n\n        initial_args = super(ConjugateGradient, self).tf_initialize(x_init)\n\n        # r_0 := b - A * x_0\n        # c_0 := r_0\n        conjugate = residual = [t - fx for t, fx in zip(b, self.fn_x(x_init))]\n\n        # r_0^2 := r^T * r\n        squared_residual = tf.add_n(inputs=[tf.reduce_sum(input_tensor=(res * res)) for res in residual])\n\n        return initial_args + (conjugate, residual, squared_residual)", "response": "This function is called by tf_step_x_init when the first iteration of the loop body is called."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a term that is true if another iteration should be performed.", "response": "def tf_next_step(self, x, iteration, conjugate, residual, squared_residual):\n        \"\"\"\n        Termination condition: max number of iterations, or residual sufficiently small.\n\n        Args:\n            x: Current solution estimate $x_t$.\n            iteration: Current iteration counter $t$.\n            conjugate: Current conjugate $c_t$.\n            residual: Current residual $r_t$.\n            squared_residual: Current squared residual $r_t^2$.\n\n        Returns:\n            True if another iteration should be performed.\n        \"\"\"\n        next_step = super(ConjugateGradient, self).tf_next_step(x, iteration, conjugate, residual, squared_residual)\n        return tf.logical_and(x=next_step, y=(squared_residual >= util.epsilon))"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates the TensorFlow operations for performing an optimization step.", "response": "def tf_step(self, time, variables, **kwargs):\n        \"\"\"\n        Creates the TensorFlow operations for performing an optimization step.\n\n        Args:\n            time: Time tensor.\n            variables: List of variables to optimize.\n            **kwargs: Additional arguments passed on to the internal optimizer.\n\n        Returns:\n            List of delta tensors corresponding to the updates for each optimized variable.\n        \"\"\"\n        deltas = self.optimizer.step(time=time, variables=variables, **kwargs)\n\n        with tf.control_dependencies(control_inputs=deltas):\n            clipped_deltas = list()\n            exceeding_deltas = list()\n            for delta in deltas:\n                clipped_delta = tf.clip_by_value(\n                    t=delta,\n                    clip_value_min=-self.clipping_value,\n                    clip_value_max=self.clipping_value\n                )\n                clipped_deltas.append(clipped_delta)\n                exceeding_deltas.append(clipped_delta - delta)\n\n        applied = self.apply_step(variables=variables, deltas=exceeding_deltas)\n\n        with tf.control_dependencies(control_inputs=(applied,)):\n            return [delta + 0.0 for delta in clipped_deltas]"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef from_spec(spec, kwargs=None):\n        layer = util.get_object(\n            obj=spec,\n            predefined_objects=tensorforce.core.networks.layers,\n            kwargs=kwargs\n        )\n        assert isinstance(layer, Layer)\n        return layer", "response": "Creates a layer from a specification dict."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate the deltas for the action based on the given Q values.", "response": "def tf_q_delta(self, q_value, next_q_value, terminal, reward):\n        \"\"\"\n        Creates the deltas (or advantage) of the Q values.\n\n        :return: A list of deltas per action\n        \"\"\"\n        for _ in range(util.rank(q_value) - 1):\n            terminal = tf.expand_dims(input=terminal, axis=1)\n            reward = tf.expand_dims(input=reward, axis=1)\n\n        multiples = (1,) + util.shape(q_value)[1:]\n        terminal = tf.tile(input=terminal, multiples=multiples)\n        reward = tf.tile(input=reward, multiples=multiples)\n\n        zeros = tf.zeros_like(tensor=next_q_value)\n        next_q_value = tf.where(condition=terminal, x=zeros, y=(self.discount * next_q_value))\n\n        return reward + next_q_value - q_value"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef target_optimizer_arguments(self):\n        variables = self.target_network.get_variables() + [\n            variable for name in sorted(self.target_distributions)\n            for variable in self.target_distributions[name].get_variables()\n        ]\n        source_variables = self.network.get_variables() + [\n            variable for name in sorted(self.distributions)\n            for variable in self.distributions[name].get_variables()\n        ]\n        arguments = dict(\n            time=self.global_timestep,\n            variables=variables,\n            source_variables=source_variables\n        )\n        if self.global_model is not None:\n            arguments['global_variables'] = self.global_model.target_network.get_variables() + [\n                variable for name in sorted(self.global_model.target_distributions)\n                for variable in self.global_model.target_distributions[name].get_variables()\n            ]\n        return arguments", "response": "Returns the target optimizer arguments as a dict."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates an Environment object from a specification dict.", "response": "def from_spec(spec, kwargs):\n        \"\"\"\n        Creates an environment from a specification dict.\n        \"\"\"\n        env = tensorforce.util.get_object(\n            obj=spec,\n            predefined_objects=tensorforce.environments.environments,\n            kwargs=kwargs\n        )\n        assert isinstance(env, Environment)\n        return env"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef setup(app):\n    global _is_sphinx\n    _is_sphinx = True\n    app.add_config_value('no_underscore_emphasis', False, 'env')\n    app.add_source_parser('.md', M2RParser)\n    app.add_directive('mdinclude', MdInclude)", "response": "When used for spinx extension."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\npasses through rest role.", "response": "def output_image_link(self, m):\n        \"\"\"Pass through rest role.\"\"\"\n        return self.renderer.image_link(\n            m.group('url'), m.group('target'), m.group('alt'))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef output_eol_literal_marker(self, m):\n        marker = ':' if m.group(1) is None else ''\n        return self.renderer.eol_literal_marker(marker)", "response": "Pass through rest link."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nrenders list tags like ul and ol.", "response": "def list(self, body, ordered=True):\n        \"\"\"Rendering list tags like ``<ul>`` and ``<ol>``.\n\n        :param body: body contents of the list.\n        :param ordered: whether this list is ordered or not.\n        \"\"\"\n        mark = '#. ' if ordered else '* '\n        lines = body.splitlines()\n        for i, line in enumerate(lines):\n            if line and not line.startswith(self.list_marker):\n                lines[i] = ' ' * len(mark) + line\n        return '\\n{}\\n'.format(\n            '\\n'.join(lines)).replace(self.list_marker, mark)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nrendering table element. Wrap header and body in it.", "response": "def table(self, header, body):\n        \"\"\"Rendering table element. Wrap header and body in it.\n\n        :param header: header part of the table.\n        :param body: body part of the table.\n        \"\"\"\n        table = '\\n.. list-table::\\n'\n        if header and not header.isspace():\n            table = (table + self.indent + ':header-rows: 1\\n\\n' +\n                     self._indent_block(header) + '\\n')\n        else:\n            table = table + '\\n'\n        table = table + self._indent_block(body) + '\\n\\n'\n        return table"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef table_row(self, content):\n        contents = content.splitlines()\n        if not contents:\n            return ''\n        clist = ['* ' + contents[0]]\n        if len(contents) > 1:\n            for c in contents[1:]:\n                clist.append('  ' + c)\n        return '\\n'.join(clist) + '\\n'", "response": "Rendering a table row. Like <tr >."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef codespan(self, text):\n        if '``' not in text:\n            return '\\ ``{}``\\ '.format(text)\n        else:\n            # actually, docutils split spaces in literal\n            return self._raw_html(\n                '<code class=\"docutils literal\">'\n                '<span class=\"pre\">{}</span>'\n                '</code>'.format(text.replace('`', '&#96;')))", "response": "Rendering inline code text."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nrendering a given link with content and title.", "response": "def link(self, link, title, text):\n        \"\"\"Rendering a given link with content and title.\n\n        :param link: href link for ``<a>`` tag.\n        :param title: title content for `title` attribute.\n        :param text: text content for description.\n        \"\"\"\n        if title:\n            raise NotImplementedError('sorry')\n        return '\\ `{text} <{target}>`_\\ '.format(target=link, text=text)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrendering an image with title and text.", "response": "def image(self, src, title, text):\n        \"\"\"Rendering a image with title and text.\n\n        :param src: source link of the image.\n        :param title: title text of the image.\n        :param text: alt text of the image.\n        \"\"\"\n        # rst does not support title option\n        # and I couldn't find title attribute in HTML standard\n        return '\\n'.join([\n            '',\n            '.. image:: {}'.format(src),\n            '   :target: {}'.format(src),\n            '   :alt: {}'.format(text),\n            '',\n        ])"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nparse the next line of the file and return a new instance of the class.", "response": "def run(self):\n        \"\"\"Most of this method is from ``docutils.parser.rst.Directive``.\n\n        docutils version: 0.12\n        \"\"\"\n        if not self.state.document.settings.file_insertion_enabled:\n            raise self.warning('\"%s\" directive disabled.' % self.name)\n        source = self.state_machine.input_lines.source(\n            self.lineno - self.state_machine.input_offset - 1)\n        source_dir = os.path.dirname(os.path.abspath(source))\n        path = rst.directives.path(self.arguments[0])\n        path = os.path.normpath(os.path.join(source_dir, path))\n        path = utils.relative_path(None, path)\n        path = nodes.reprunicode(path)\n\n        # get options (currently not use directive-specific options)\n        encoding = self.options.get(\n            'encoding', self.state.document.settings.input_encoding)\n        e_handler = self.state.document.settings.input_encoding_error_handler\n        tab_width = self.options.get(\n            'tab-width', self.state.document.settings.tab_width)\n\n        # open the inclding file\n        try:\n            self.state.document.settings.record_dependencies.add(path)\n            include_file = io.FileInput(source_path=path,\n                                        encoding=encoding,\n                                        error_handler=e_handler)\n        except UnicodeEncodeError as error:\n            raise self.severe('Problems with \"%s\" directive path:\\n'\n                              'Cannot encode input file path \"%s\" '\n                              '(wrong locale?).' %\n                              (self.name, SafeString(path)))\n        except IOError as error:\n            raise self.severe('Problems with \"%s\" directive path:\\n%s.' %\n                              (self.name, ErrorString(error)))\n\n        # read from the file\n        try:\n            rawtext = include_file.read()\n        except UnicodeError as error:\n            raise self.severe('Problem with \"%s\" directive:\\n%s' %\n                              (self.name, ErrorString(error)))\n\n        config = self.state.document.settings.env.config\n        converter = M2R(no_underscore_emphasis=config.no_underscore_emphasis)\n        include_lines = statemachine.string2lines(converter(rawtext),\n                                                  tab_width,\n                                                  convert_whitespace=True)\n        self.state_machine.insert_input(include_lines, path)\n        return []"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nclone a given Agent and returns a list of the original Agent s in the first slot.", "response": "def clone_worker_agent(agent, factor, environment, network, agent_config):\n    \"\"\"\n    Clones a given Agent (`factor` times) and returns a list of the cloned Agents with the original Agent\n    in the first slot.\n\n    Args:\n        agent (Agent): The Agent object to clone.\n        factor (int): The length of the final list.\n        environment (Environment): The Environment to use for all cloned agents.\n        network (LayeredNetwork): The Network to use (or None) for an Agent's Model.\n        agent_config (dict): A dict of Agent specifications passed into the Agent's c'tor as kwargs.\n    Returns:\n        The list with `factor` cloned agents (including the original one).\n    \"\"\"\n    ret = [agent]\n    for i in xrange(factor - 1):\n        worker = WorkerAgentGenerator(type(agent))(\n            states=environment.states,\n            actions=environment.actions,\n            network=network,\n            model=agent.model,\n            **agent_config\n        )\n        ret.append(worker)\n\n    return ret"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nexecuting this runner by starting all Agents in parallel (each one in one thread). Args: episodes (int): Deprecated; see num_episodes. max_timesteps (int): Deprecated; see max_episode_timesteps.", "response": "def run(\n        self,\n        num_episodes=-1,\n        max_episode_timesteps=-1,\n        episode_finished=None,\n        summary_report=None,\n        summary_interval=0,\n        num_timesteps=None,\n        deterministic=False,\n        episodes=None,\n        max_timesteps=None,\n        testing=False,\n        sleep=None\n    ):\n        \"\"\"\n        Executes this runner by starting all Agents in parallel (each one in one thread).\n\n        Args:\n            episodes (int): Deprecated; see num_episodes.\n            max_timesteps (int): Deprecated; see max_episode_timesteps.\n        \"\"\"\n\n        # Renamed episodes into num_episodes to match BaseRunner's signature (fully backw. compatible).\n        if episodes is not None:\n            num_episodes = episodes\n            warnings.warn(\"WARNING: `episodes` parameter is deprecated, use `num_episodes` instead.\",\n                          category=DeprecationWarning)\n        assert isinstance(num_episodes, int)\n        # Renamed max_timesteps into max_episode_timesteps to match single Runner's signature (fully backw. compatible).\n        if max_timesteps is not None:\n            max_episode_timesteps = max_timesteps\n            warnings.warn(\"WARNING: `max_timesteps` parameter is deprecated, use `max_episode_timesteps` instead.\",\n                          category=DeprecationWarning)\n        assert isinstance(max_episode_timesteps, int)\n\n        if summary_report is not None:\n            warnings.warn(\"WARNING: `summary_report` parameter is deprecated, use `episode_finished` callback \"\n                          \"instead to generate summaries every n episodes.\",\n                          category=DeprecationWarning)\n\n        self.reset()\n\n        # Reset counts/stop-condition for this run.\n        self.global_episode = 0\n        self.global_timestep = 0\n        self.should_stop = False\n\n        # Create threads.\n        threads = [threading.Thread(target=self._run_single, args=(t, self.agent[t], self.environment[t],),\n                                    kwargs={\"deterministic\": deterministic,\n                                            \"max_episode_timesteps\": max_episode_timesteps,\n                                            \"episode_finished\": episode_finished,\n                                            \"testing\": testing,\n                                            \"sleep\": sleep})\n                   for t in range(len(self.agent))]\n\n        # Start threads.\n        self.start_time = time.time()\n        [t.start() for t in threads]\n\n        # Stay idle until killed by SIGINT or a global stop condition is met.\n        try:\n            next_summary = 0\n            next_save = 0 if self.save_frequency_unit != \"s\" else time.time()\n            while any([t.is_alive() for t in threads]) and self.global_episode < num_episodes or num_episodes == -1:\n                self.time = time.time()\n\n                # This is deprecated (but still supported) and should be covered by the `episode_finished` callable.\n                if summary_report is not None and self.global_episode > next_summary:\n                    summary_report(self)\n                    next_summary += summary_interval\n\n                if self.save_path and self.save_frequency is not None:\n                    do_save = True\n                    current = None\n                    if self.save_frequency_unit == \"e\" and self.global_episode > next_save:\n                        current = self.global_episode\n                    elif self.save_frequency_unit == \"s\" and self.time > next_save:\n                        current = self.time\n                    elif self.save_frequency_unit == \"t\" and self.global_timestep > next_save:\n                        current = self.global_timestep\n                    else:\n                        do_save = False\n\n                    if do_save:\n                        self.agent[0].save_model(self.save_path)\n                        # Make sure next save is later than right now.\n                        while next_save < current:\n                            next_save += self.save_frequency\n                time.sleep(1)\n\n        except KeyboardInterrupt:\n            print('Keyboard interrupt, sending stop command to threads')\n\n        self.should_stop = True\n\n        # Join threads.\n        [t.join() for t in threads]\n        print('All threads stopped')"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert integer to x y coordinates.", "response": "def _int_to_pos(self, flat_position):\n        \"\"\"Returns x, y from flat_position integer.\n\n        Args:\n            flat_position: flattened position integer\n\n        Returns: x, y\n\n        \"\"\"\n        return flat_position % self.env.action_space.screen_shape[0],\\\n            flat_position % self.env.action_space.screen_shape[1]"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _wait_state(self, state, reward, terminal):\n        while state == [None] or not state:\n             state, terminal, reward = self._execute(dict(key=0))\n\n        return state, terminal, reward", "response": "Wait until the state is in the specified state."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\napply the given step deltas to the variable values.", "response": "def apply_step(self, variables, deltas):\n        \"\"\"\n        Applies the given (and already calculated) step deltas to the variable values.\n\n        Args:\n            variables: List of variables.\n            deltas: List of deltas of same length.\n\n        Returns:\n            The step-applied operation. A tf.group of tf.assign_add ops.\n        \"\"\"\n        if len(variables) != len(deltas):\n            raise TensorForceError(\"Invalid variables and deltas lists.\")\n        return tf.group(\n            *(tf.assign_add(ref=variable, value=delta) for variable, delta in zip(variables, deltas))\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nperforming an optimization step.", "response": "def minimize(self, time, variables, **kwargs):\n        \"\"\"\n        Performs an optimization step.\n\n        Args:\n            time: Time tensor.\n            variables: List of variables to optimize.\n            **kwargs: Additional optimizer-specific arguments. The following arguments are used\n                by some optimizers:\n            - arguments: Dict of arguments for callables, like fn_loss.\n            - fn_loss: A callable returning the loss of the current model.\n            - fn_reference: A callable returning the reference values, in case of a comparative  \n                loss.\n            - fn_kl_divergence: A callable returning the KL-divergence relative to the\n                current model.\n            - sampled_loss: A sampled loss (integer).\n            - return_estimated_improvement: Returns the estimated improvement resulting from\n                the natural gradient calculation if true.\n            - source_variables: List of source variables to synchronize with.\n            - global_variables: List of global variables to apply the proposed optimization\n                step to.\n\n\n        Returns:\n            The optimization operation.\n        \"\"\"\n        # # Add training variable gradient histograms/scalars to summary output\n        # # if 'gradients' in self.summary_labels:\n        # if any(k in self.summary_labels for k in ['gradients', 'gradients_histogram', 'gradients_scalar']):\n        #     valid = True\n        #     if isinstance(self, tensorforce.core.optimizers.TFOptimizer):\n        #         gradients = self.optimizer.compute_gradients(kwargs['fn_loss']())\n        #     elif isinstance(self.optimizer, tensorforce.core.optimizers.TFOptimizer):\n        #         # This section handles \"Multi_step\" and may handle others\n        #         # if failure is found, add another elif to handle that case\n        #         gradients = self.optimizer.optimizer.compute_gradients(kwargs['fn_loss']())\n        #     else:\n        #         # Didn't find proper gradient information\n        #         valid = False\n\n        #     # Valid gradient data found, create summary data items\n        #     if valid:\n        #         for grad, var in gradients:\n        #             if grad is not None:\n        #                 if any(k in self.summary_labels for k in ('gradients', 'gradients_scalar')):\n        #                     axes = list(range(len(grad.shape)))\n        #                     mean, var = tf.nn.moments(grad, axes)\n        #                     tf.contrib.summary.scalar(name='gradients/' + var.name + \"/mean\", tensor=mean)\n        #                     tf.contrib.summary.scalar(name='gradients/' + var.name + \"/variance\", tensor=var)\n        #                 if any(k in self.summary_labels for k in ('gradients', 'gradients_histogram')):\n        #                     tf.contrib.summary.histogram(name='gradients/' + var.name, tensor=grad)\n\n        deltas = self.step(time=time, variables=variables, **kwargs)\n        with tf.control_dependencies(control_inputs=deltas):\n            return tf.no_op()"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef from_spec(spec, kwargs=None):\n        optimizer = util.get_object(\n            obj=spec,\n            predefined_objects=tensorforce.core.optimizers.optimizers,\n            kwargs=kwargs\n        )\n        assert isinstance(optimizer, Optimizer)\n        return optimizer", "response": "Creates an optimizer from a dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef np_dtype(dtype):\n    if dtype == 'float' or dtype == float or dtype == np.float32 or dtype == tf.float32:\n        return np.float32\n    elif dtype == np.float64 or dtype == tf.float64:\n        return np.float64\n    elif dtype == np.float16 or dtype == tf.float16:\n        return np.float16\n    elif dtype == 'int' or dtype == int or dtype == np.int32 or dtype == tf.int32:\n        return np.int32\n    elif dtype == np.int64 or dtype == tf.int64:\n        return np.int64\n    elif dtype == np.int16 or dtype == tf.int16:\n        return np.int16\n    elif dtype == 'bool' or dtype == bool or dtype == np.bool_ or dtype == tf.bool:\n        return np.bool_\n    else:\n        raise TensorForceError(\"Error: Type conversion from type {} not supported.\".format(str(dtype)))", "response": "Translates dtype specifications in configurations to Numpy data types."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_tensor_dependencies(tensor):\n    dependencies = set()\n    dependencies.update(tensor.op.inputs)\n    for sub_op in tensor.op.inputs:\n        dependencies.update(get_tensor_dependencies(sub_op))\n    return dependencies", "response": "Utility method to get all dependencies of a tensor."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_object(obj, predefined_objects=None, default_object=None, kwargs=None):\n    args = ()\n    kwargs = dict() if kwargs is None else kwargs\n\n    if isinstance(obj, str) and os.path.isfile(obj):\n        with open(obj, 'r') as fp:\n            obj = json.load(fp=fp)\n    if isinstance(obj, dict):\n        kwargs.update(obj)\n        obj = kwargs.pop('type', None)\n\n    if predefined_objects is not None and obj in predefined_objects:\n        obj = predefined_objects[obj]\n    elif isinstance(obj, str):\n        if obj.find('.') != -1:\n            module_name, function_name = obj.rsplit('.', 1)\n            module = importlib.import_module(module_name)\n            obj = getattr(module, function_name)\n        else:\n            raise TensorForceError(\"Error: object {} not found in predefined objects: {}\".format(\n                obj,\n                list(predefined_objects or ())\n            ))\n    elif callable(obj):\n        pass\n    elif default_object is not None:\n        args = (obj,)\n        obj = default_object\n    else:\n        # assumes the object is already instantiated\n        return obj\n\n    return obj(*args, **kwargs)", "response": "Utility method to map some kind of object specification to its content."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nregisters the saver operations to the graph in context.", "response": "def register_saver_ops(self):\n        \"\"\"\n        Registers the saver operations to the graph in context.\n        \"\"\"\n\n        variables = self.get_savable_variables()\n        if variables is None or len(variables) == 0:\n            self._saver = None\n            return\n\n        base_scope = self._get_base_variable_scope()\n        variables_map = {strip_name_scope(v.name, base_scope): v for v in variables}\n\n        self._saver = tf.train.Saver(\n            var_list=variables_map,\n            reshape=False,\n            sharded=False,\n            max_to_keep=5,\n            keep_checkpoint_every_n_hours=10000.0,\n            name=None,\n            restore_sequentially=False,\n            saver_def=None,\n            builder=None,\n            defer_build=False,\n            allow_empty=True,\n            write_version=tf.train.SaverDef.V2,\n            pad_step_number=False,\n            save_relative_paths=True\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsaving the current state of the component.", "response": "def save(self, sess, save_path, timestep=None):\n        \"\"\"\n        Saves this component's managed variables.\n\n        Args:\n            sess: The session for which to save the managed variables.\n            save_path: The path to save data to.\n            timestep: Optional, the timestep to append to the file name.\n\n        Returns:\n            Checkpoint path where the model was saved.\n        \"\"\"\n\n        if self._saver is None:\n            raise TensorForceError(\"register_saver_ops should be called before save\")\n        return self._saver.save(\n            sess=sess,\n            save_path=save_path,\n            global_step=timestep,\n            write_meta_graph=False,\n            write_state=True,  # Do we need this?\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef restore(self, sess, save_path):\n\n        if self._saver is None:\n            raise TensorForceError(\"register_saver_ops should be called before restore\")\n        self._saver.restore(sess=sess, save_path=save_path)", "response": "Restores the values of the managed variables from disk location."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef reset(self):\n        fetches = []\n        for processor in self.preprocessors:\n            fetches.extend(processor.reset() or [])\n        return fetches", "response": "Calls reset on all our preprocessors and returns a list of tensors to be fetched."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef process(self, tensor):\n        for processor in self.preprocessors:\n            tensor = processor.process(tensor=tensor)\n        return tensor", "response": "Process a single state."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the original shape of preprocessed state given original shape.", "response": "def processed_shape(self, shape):\n        \"\"\"\n        Shape of preprocessed state given original shape.\n\n        Args:\n            shape: original state shape\n\n        Returns: processed state shape\n        \"\"\"\n        for processor in self.preprocessors:\n            shape = processor.processed_shape(shape=shape)\n        return shape"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef from_spec(spec, kwargs=None):\n        if isinstance(spec, dict):\n            spec = [spec]\n\n        stack = PreprocessorStack()\n        for preprocessor_spec in spec:\n            # need to deep copy, otherwise will add first processors spec_ to kwargs to second processor\n            preprocessor_kwargs = copy.deepcopy(kwargs)\n            preprocessor = util.get_object(\n                obj=preprocessor_spec,\n                predefined_objects=tensorforce.core.preprocessors.preprocessors,\n                kwargs=preprocessor_kwargs\n            )\n            assert isinstance(preprocessor, Preprocessor)\n            stack.preprocessors.append(preprocessor)\n\n        return stack", "response": "Creates a preprocessing stack from a specification dict."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nexecutes action and returns next state reward done", "response": "def execute(self, action):\n        \"\"\"\n        Executes action, observes next state and reward.\n\n        Args:\n            actions: Actions to execute.\n\n        Returns:\n            Tuple of (next state, bool indicating terminal, reward)\n        \"\"\"\n        next_state, rew, done, _ = self.env.step(action)\n        return next_state, rew, done"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef as_local_model(self):\n        super(MemoryModel, self).as_local_model()\n        self.optimizer_spec = dict(\n            type='global_optimizer',\n            optimizer=self.optimizer_spec\n        )", "response": "Makes sure our optimizer is wrapped into the global_optimizer meta."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nconstruct the memory and optimizer objects and creates all template functions.", "response": "def setup_components_and_tf_funcs(self, custom_getter=None):\n        \"\"\"\n        Constructs the memory and the optimizer objects.\n        Generates and stores all template functions.\n        \"\"\"\n        custom_getter = super(MemoryModel, self).setup_components_and_tf_funcs(custom_getter)\n\n        # Memory\n        self.memory = Memory.from_spec(\n            spec=self.memory_spec,\n            kwargs=dict(\n                states=self.states_spec,\n                internals=self.internals_spec,\n                actions=self.actions_spec,\n                summary_labels=self.summary_labels\n            )\n        )\n\n        # Optimizer\n        self.optimizer = Optimizer.from_spec(\n            spec=self.optimizer_spec,\n            kwargs=dict(summary_labels=self.summary_labels)\n        )\n\n        # TensorFlow functions\n        self.fn_discounted_cumulative_reward = tf.make_template(\n            name_='discounted-cumulative-reward',\n            func_=self.tf_discounted_cumulative_reward,\n            custom_getter_=custom_getter\n        )\n        self.fn_reference = tf.make_template(\n            name_='reference',\n            func_=self.tf_reference,\n            custom_getter_=custom_getter\n        )\n        self.fn_loss_per_instance = tf.make_template(\n            name_='loss-per-instance',\n            func_=self.tf_loss_per_instance,\n            custom_getter_=custom_getter\n        )\n        self.fn_regularization_losses = tf.make_template(\n            name_='regularization-losses',\n            func_=self.tf_regularization_losses,\n            custom_getter_=custom_getter\n        )\n        self.fn_loss = tf.make_template(\n            name_='loss',\n            func_=self.tf_loss,\n            custom_getter_=custom_getter\n        )\n        self.fn_optimization = tf.make_template(\n            name_='optimization',\n            func_=self.tf_optimization,\n            custom_getter_=custom_getter\n        )\n        self.fn_import_experience = tf.make_template(\n            name_='import-experience',\n            func_=self.tf_import_experience,\n            custom_getter_=custom_getter\n        )\n\n        return custom_getter"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef tf_discounted_cumulative_reward(self, terminal, reward, discount=None, final_reward=0.0, horizon=0):\n\n        # By default -> take Model's gamma value\n        if discount is None:\n            discount = self.discount\n\n        # Accumulates discounted (n-step) reward (start new if terminal)\n        def cumulate(cumulative, reward_terminal_horizon_subtract):\n            rew, is_terminal, is_over_horizon, sub = reward_terminal_horizon_subtract\n            return tf.where(\n                # If terminal, start new cumulation.\n                condition=is_terminal,\n                x=rew,\n                y=tf.where(\n                    # If we are above the horizon length (H) -> subtract discounted value from H steps back.\n                    condition=is_over_horizon,\n                    x=(rew + cumulative * discount - sub),\n                    y=(rew + cumulative * discount)\n                )\n            )\n\n        # Accumulates length of episodes (starts new if terminal)\n        def len_(cumulative, term):\n            return tf.where(\n                condition=term,\n                # Start counting from 1 after is-terminal signal\n                x=tf.ones(shape=(), dtype=tf.int32),\n                # Otherwise, increase length by 1\n                y=cumulative + 1\n            )\n\n        # Reverse, since reward cumulation is calculated right-to-left, but tf.scan only works left-to-right.\n        reward = tf.reverse(tensor=reward, axis=(0,))\n        # e.g. -1.0 1.0 0.5 0.0 1.0 2.0\n        terminal = tf.reverse(tensor=terminal, axis=(0,))\n        # e.g. F T F F F F\n\n        # Store the steps until end of the episode(s) determined by the input terminal signals (True starts new count).\n        lengths = tf.scan(fn=len_, elems=terminal, initializer=0)\n        # e.g. 1 1 2 3 4 5\n        off_horizon = tf.greater(lengths, tf.fill(dims=tf.shape(lengths), value=horizon))\n        # e.g. F F F F T T\n\n        # Calculate the horizon-subtraction value for each step.\n        if horizon > 0:\n            horizon_subtractions = tf.map_fn(lambda x: (discount ** horizon) * x, reward, dtype=tf.float32)\n            # Shift right by size of horizon (fill rest with 0.0).\n            horizon_subtractions = tf.concat([np.zeros(shape=(horizon,)), horizon_subtractions], axis=0)\n            horizon_subtractions = tf.slice(horizon_subtractions, begin=(0,), size=tf.shape(reward))\n            # e.g. 0.0, 0.0, 0.0, -1.0*g^3, 1.0*g^3, 0.5*g^3\n        # all 0.0 if infinite horizon (special case: horizon=0)\n        else:\n            horizon_subtractions = tf.zeros(shape=tf.shape(reward))\n\n        # Now do the scan, each time summing up the previous step (discounted by gamma) and\n        # subtracting the respective `horizon_subtraction`.\n        reward = tf.scan(\n            fn=cumulate,\n            elems=(reward, terminal, off_horizon, horizon_subtractions),\n            initializer=final_reward if horizon != 1 else 0.0\n        )\n        # Re-reverse again to match input sequences.\n        return tf.reverse(tensor=reward, axis=(0,))", "response": "Creates and returns a TensorFlow operation for calculating the discounted cumulative reward for a given sequence of single rewards."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a TensorFlow operation for obtaining the reference tensor for the given state internals actions terminal reward next_states next_internals update.", "response": "def tf_reference(self, states, internals, actions, terminal, reward, next_states, next_internals, update):\n        \"\"\"\n        Creates the TensorFlow operations for obtaining the reference tensor(s), in case of a\n        comparative loss.\n\n        Args:\n            states: Dict of state tensors.\n            internals: List of prior internal state tensors.\n            actions: Dict of action tensors.\n            terminal: Terminal boolean tensor.\n            reward: Reward tensor.\n            next_states: Dict of successor state tensors.\n            next_internals: List of posterior internal state tensors.\n            update: Boolean tensor indicating whether this call happens during an update.\n\n        Returns:\n            Reference tensor(s).\n        \"\"\"\n        return None"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the TensorFlow operations for calculating the loss per instance.", "response": "def tf_loss_per_instance(self, states, internals, actions, terminal, reward,\n                             next_states, next_internals, update, reference=None):\n        \"\"\"\n        Creates the TensorFlow operations for calculating the loss per batch instance.\n\n        Args:\n            states: Dict of state tensors.\n            internals: Dict of prior internal state tensors.\n            actions: Dict of action tensors.\n            terminal: Terminal boolean tensor.\n            reward: Reward tensor.\n            next_states: Dict of successor state tensors.\n            next_internals: List of posterior internal state tensors.\n            update: Boolean tensor indicating whether this call happens during an update.\n            reference: Optional reference tensor(s), in case of a comparative loss.\n\n        Returns:\n            Loss per instance tensor.\n        \"\"\"\n        raise NotImplementedError"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the TensorFlow loss for a single internal state.", "response": "def tf_loss(self, states, internals, actions, terminal, reward, next_states, next_internals, update, reference=None):\n        \"\"\"\n        Creates the TensorFlow operations for calculating the full loss of a batch.\n\n        Args:\n            states: Dict of state tensors.\n            internals: List of prior internal state tensors.\n            actions: Dict of action tensors.\n            terminal: Terminal boolean tensor.\n            reward: Reward tensor.\n            next_states: Dict of successor state tensors.\n            next_internals: List of posterior internal state tensors.\n            update: Boolean tensor indicating whether this call happens during an update.\n            reference: Optional reference tensor(s), in case of a comparative loss.\n\n        Returns:\n            Loss tensor.\n        \"\"\"\n        # Mean loss per instance\n        loss_per_instance = self.fn_loss_per_instance(\n            states=states,\n            internals=internals,\n            actions=actions,\n            terminal=terminal,\n            reward=reward,\n            next_states=next_states,\n            next_internals=next_internals,\n            update=update,\n            reference=reference\n        )\n\n        # Returns no-op.\n        updated = self.memory.update_batch(loss_per_instance=loss_per_instance)\n        with tf.control_dependencies(control_inputs=(updated,)):\n            loss = tf.reduce_mean(input_tensor=loss_per_instance, axis=0)\n\n            # Loss without regularization summary.\n            if 'losses' in self.summary_labels:\n                tf.contrib.summary.scalar(name='loss-without-regularization', tensor=loss)\n\n            # Regularization losses.\n            losses = self.fn_regularization_losses(states=states, internals=internals, update=update)\n            if len(losses) > 0:\n                loss += tf.add_n(inputs=[losses[name] for name in sorted(losses)])\n                if 'regularization' in self.summary_labels:\n                    for name in sorted(losses):\n                        tf.contrib.summary.scalar(name=('regularization/' + name), tensor=losses[name])\n\n            # Total loss summary.\n            if 'losses' in self.summary_labels or 'total-loss' in self.summary_labels:\n                tf.contrib.summary.scalar(name='total-loss', tensor=loss)\n\n            return loss"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the arguments to be used as keyword arguments to the optimizer.", "response": "def optimizer_arguments(self, states, internals, actions, terminal, reward, next_states, next_internals):\n        \"\"\"\n        Returns the optimizer arguments including the time, the list of variables to optimize,\n        and various functions which the optimizer might require to perform an update step.\n\n        Args:\n            states (dict): Dict of state tensors.\n            internals (dict): Dict of prior internal state tensors.\n            actions (dict): Dict of action tensors.\n            terminal: 1D boolean is-terminal tensor.\n            reward: 1D (float) rewards tensor.\n            next_states (dict): Dict of successor state tensors.\n            next_internals (dict): Dict of posterior internal state tensors.\n\n        Returns:\n            Optimizer arguments as dict to be used as **kwargs to the optimizer.\n        \"\"\"\n        arguments = dict(\n            time=self.global_timestep,\n            variables=self.get_variables(),\n            arguments=dict(\n                states=states,\n                internals=internals,\n                actions=actions,\n                terminal=terminal,\n                reward=reward,\n                next_states=next_states,\n                next_internals=next_internals,\n                update=tf.constant(value=True)\n            ),\n            fn_reference=self.fn_reference,\n            fn_loss=self.fn_loss\n        )\n        if self.global_model is not None:\n            arguments['global_variables'] = self.global_model.get_variables()\n        return arguments"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates the TensorFlow operations for performing an optimization update step based on the given input states and actions batch. Args: states: Dict of state tensors. internals: List of prior internal state tensors. actions: Dict of action tensors. terminal: Terminal boolean tensor. reward: Reward tensor. next_states: Dict of successor state tensors. next_internals: List of posterior internal state tensors. Returns: The optimization operation.", "response": "def tf_optimization(self, states, internals, actions, terminal, reward, next_states=None, next_internals=None):\n        \"\"\"\n        Creates the TensorFlow operations for performing an optimization update step based\n        on the given input states and actions batch.\n\n        Args:\n            states: Dict of state tensors.\n            internals: List of prior internal state tensors.\n            actions: Dict of action tensors.\n            terminal: Terminal boolean tensor.\n            reward: Reward tensor.\n            next_states: Dict of successor state tensors.\n            next_internals: List of posterior internal state tensors.\n\n        Returns:\n            The optimization operation.\n        \"\"\"\n        arguments = self.optimizer_arguments(\n            states=states,\n            internals=internals,\n            actions=actions,\n            terminal=terminal,\n            reward=reward,\n            next_states=next_states,\n            next_internals=next_internals\n        )\n        return self.optimizer.minimize(**arguments)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate and returns the op that observes a timestep.", "response": "def tf_observe_timestep(self, states, internals, actions, terminal, reward):\n        \"\"\"\n        Creates and returns the op that - if frequency condition is hit - pulls a batch from the memory\n        and does one optimization step.\n        \"\"\"\n        # Store timestep in memory\n        stored = self.memory.store(\n            states=states,\n            internals=internals,\n            actions=actions,\n            terminal=terminal,\n            reward=reward\n        )\n\n        # Periodic optimization\n        with tf.control_dependencies(control_inputs=(stored,)):\n            unit = self.update_mode['unit']\n            batch_size = self.update_mode['batch_size']\n            frequency = self.update_mode.get('frequency', batch_size)\n            first_update = self.update_mode.get('first_update', 0)\n\n            if unit == 'timesteps':\n                # Timestep-based batch\n                optimize = tf.logical_and(\n                    x=tf.equal(x=(self.timestep % frequency), y=0),\n                    y=tf.logical_and(\n                        x=tf.greater_equal(x=self.timestep, y=batch_size),\n                        y=tf.greater_equal(x=self.timestep, y=first_update)\n                    )\n                )\n\n            elif unit == 'episodes':\n                # Episode-based batch\n                optimize = tf.logical_and(\n                    x=tf.equal(x=(self.episode % frequency), y=0),\n                    y=tf.logical_and(\n                        # Only update once per episode increment.\n                        x=tf.greater(x=tf.count_nonzero(input_tensor=terminal), y=0),\n                        y=tf.logical_and(\n                            x=tf.greater_equal(x=self.episode, y=batch_size),\n                            y=tf.greater_equal(x=self.episode, y=first_update)\n                        )\n                    )\n                )\n\n            elif unit == 'sequences':\n                # Timestep-sequence-based batch\n                sequence_length = self.update_mode.get('length', 8)\n                optimize = tf.logical_and(\n                    x=tf.equal(x=(self.timestep % frequency), y=0),\n                    y=tf.logical_and(\n                        x=tf.greater_equal(x=self.timestep, y=(batch_size + sequence_length - 1)),\n                        y=tf.greater_equal(x=self.timestep, y=first_update)\n                    )\n                )\n\n            else:\n                raise TensorForceError(\"Invalid update unit: {}.\".format(unit))\n\n            def true_fn():\n                if unit == 'timesteps':\n                    # Timestep-based batch\n                    batch = self.memory.retrieve_timesteps(n=batch_size)\n                elif unit == 'episodes':\n                    # Episode-based batch\n                    batch = self.memory.retrieve_episodes(n=batch_size)\n                elif unit == 'sequences':\n                    # Timestep-sequence-based batch\n                    batch = self.memory.retrieve_sequences(n=batch_size, sequence_length=sequence_length)\n\n                # Do not calculate gradients for memory-internal operations.\n                batch = util.map_tensors(\n                    fn=(lambda tensor: tf.stop_gradient(input=tensor)),\n                    tensors=batch\n                )\n\n                optimize = self.fn_optimization(**batch)\n                with tf.control_dependencies(control_inputs=(optimize,)):\n                    return tf.logical_and(x=True, y=True)\n\n            return tf.cond(pred=optimize, true_fn=true_fn, false_fn=tf.no_op)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nimports experiences into the TensorFlow memory structure.", "response": "def tf_import_experience(self, states, internals, actions, terminal, reward):\n        \"\"\"\n        Imports experiences into the TensorFlow memory structure. Can be used to import\n        off-policy data.\n\n        :param states: Dict of state values to import with keys as state names and values as values to set.\n        :param internals: Internal values to set, can be fetched from agent via agent.current_internals\n            if no values available.\n        :param actions: Dict of action values to import with keys as action names and values as values to set.\n        :param terminal: Terminal value(s)\n        :param reward: Reward value(s)\n        \"\"\"\n        return self.memory.store(\n            states=states,\n            internals=internals,\n            actions=actions,\n            terminal=terminal,\n            reward=reward\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating the TensorFlow operations for performing an optimization step. Args: time: Time tensor. variables: List of variables to optimize. arguments: Dict of arguments for callables, like fn_loss. fn_loss: A callable returning the loss of the current model. fn_reference: A callable returning the reference values, in case of a comparative loss. **kwargs: Additional arguments passed on to the internal optimizer. Returns: List of delta tensors corresponding to the updates for each optimized variable.", "response": "def tf_step(\n        self,\n        time,\n        variables,\n        arguments,\n        fn_loss,\n        fn_reference,\n        **kwargs\n    ):\n        \"\"\"\n        Creates the TensorFlow operations for performing an optimization step.\n\n        Args:\n            time: Time tensor.\n            variables: List of variables to optimize.\n            arguments: Dict of arguments for callables, like fn_loss.\n            fn_loss: A callable returning the loss of the current model.\n            fn_reference: A callable returning the reference values, in case of a comparative loss.\n            **kwargs: Additional arguments passed on to the internal optimizer.\n\n        Returns:\n            List of delta tensors corresponding to the updates for each optimized variable.\n        \"\"\"\n\n        # Set reference to compare with at each optimization step, in case of a comparative loss.\n        arguments['reference'] = fn_reference(**arguments)\n\n        # Negative value since line search maximizes.\n        loss_before = -fn_loss(**arguments)\n\n        with tf.control_dependencies(control_inputs=(loss_before,)):\n            deltas = self.optimizer.step(\n                time=time,\n                variables=variables,\n                arguments=arguments,\n                fn_loss=fn_loss,\n                return_estimated_improvement=True,\n                **kwargs\n            )\n\n            if isinstance(deltas, tuple):\n                # If 'return_estimated_improvement' argument exists.\n                if len(deltas) != 2:\n                    raise TensorForceError(\"Unexpected output of internal optimizer.\")\n                deltas, estimated_improvement = deltas\n                # Negative value since line search maximizes.\n                estimated_improvement = -estimated_improvement\n            else:\n                estimated_improvement = None\n\n        with tf.control_dependencies(control_inputs=deltas):\n                # Negative value since line search maximizes.\n            loss_step = -fn_loss(**arguments)\n\n        with tf.control_dependencies(control_inputs=(loss_step,)):\n\n            def evaluate_step(deltas):\n                with tf.control_dependencies(control_inputs=deltas):\n                    applied = self.apply_step(variables=variables, deltas=deltas)\n                with tf.control_dependencies(control_inputs=(applied,)):\n                    # Negative value since line search maximizes.\n                    return -fn_loss(**arguments)\n\n            return self.solver.solve(\n                fn_x=evaluate_step,\n                x_init=deltas,\n                base_value=loss_before,\n                target_value=loss_step,\n                estimated_improvement=estimated_improvement\n            )"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a Distribution object from a specification dict.", "response": "def from_spec(spec, kwargs=None):\n        \"\"\"\n        Creates a distribution from a specification dict.\n        \"\"\"\n        distribution = util.get_object(\n            obj=spec,\n            predefined_objects=tensorforce.core.distributions.distributions,\n            kwargs=kwargs\n        )\n        assert isinstance(distribution, Distribution)\n        return distribution"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nresetting the agent s internal episode and time step counter and internal states.", "response": "def reset(self):\n        \"\"\"\n        Resets the agent to its initial state (e.g. on experiment start). Updates the Model's internal episode and\n        time step counter, internal states, and resets preprocessors.\n        \"\"\"\n        self.episode, self.timestep, self.next_internals = self.model.reset()\n        self.current_internals = self.next_internals"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef act(self, states, deterministic=False, independent=False, fetch_tensors=None, buffered=True, index=0):\n        self.current_internals = self.next_internals\n\n        if self.unique_state:\n            self.current_states = dict(state=np.asarray(states))\n        else:\n            self.current_states = {name: np.asarray(states[name]) for name in sorted(states)}\n\n        if fetch_tensors is not None:\n            # Retrieve action\n            self.current_actions, self.next_internals, self.timestep, self.fetched_tensors = self.model.act(\n                states=self.current_states,\n                internals=self.current_internals,\n                deterministic=deterministic,\n                independent=independent,\n                fetch_tensors=fetch_tensors,\n                index=index\n            )\n\n            if self.unique_action:\n                return self.current_actions['action'], self.fetched_tensors\n            else:\n                return self.current_actions, self.fetched_tensors\n\n        # Retrieve action.\n        self.current_actions, self.next_internals, self.timestep = self.model.act(\n            states=self.current_states,\n            internals=self.current_internals,\n            deterministic=deterministic,\n            independent=independent,\n            index=index\n        )\n\n        # Buffered mode only works single-threaded because buffer inserts\n        # by multiple threads are non-atomic and can cause race conditions.\n        if buffered:\n            if self.unique_action:\n                return self.current_actions['action']\n            else:\n                return self.current_actions\n        else:\n            if self.unique_action:\n                return self.current_actions['action'], self.current_states, self.current_internals\n            else:\n                return self.current_actions, self.current_states, self.current_internals", "response": "Returns the action of the given states."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nobserve an experience from the environment to learn from.", "response": "def observe(self, terminal, reward, index=0):\n        \"\"\"\n        Observe experience from the environment to learn from. Optionally pre-processes rewards\n        Child classes should call super to get the processed reward\n        EX: terminal, reward = super()...\n\n        Args:\n            terminal (bool): boolean indicating if the episode terminated after the observation.\n            reward (float): scalar reward that resulted from executing the action.\n        \"\"\"\n        self.current_terminal = terminal\n        self.current_reward = reward\n\n        if self.batched_observe:\n            # Batched observe for better performance with Python.\n            self.observe_terminal[index].append(self.current_terminal)\n            self.observe_reward[index].append(self.current_reward)\n\n            if self.current_terminal or len(self.observe_terminal[index]) >= self.batching_capacity:\n                self.episode = self.model.observe(\n                    terminal=self.observe_terminal[index],\n                    reward=self.observe_reward[index],\n                    index=index\n                )\n                self.observe_terminal[index] = list()\n                self.observe_reward[index] = list()\n\n        else:\n            self.episode = self.model.observe(\n                terminal=self.current_terminal,\n                reward=self.current_reward\n            )"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef atomic_observe(self, states, actions, internals, reward, terminal):\n        # TODO probably unnecessary here.\n        self.current_terminal = terminal\n        self.current_reward = reward\n        # print('action = {}'.format(actions))\n        if self.unique_state:\n            states = dict(state=states)\n        if self.unique_action:\n            actions = dict(action=actions)\n\n        self.episode = self.model.atomic_observe(\n            states=states,\n            actions=actions,\n            internals=internals,\n            terminal=self.current_terminal,\n            reward=self.current_reward\n        )", "response": "Utility method for unbuffered observing where each tuple is inserted into TensorFlow via a single session call."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsaving the current state of the current model to a checkpoint file.", "response": "def save_model(self, directory=None, append_timestep=True):\n        \"\"\"\n        Save TensorFlow model. If no checkpoint directory is given, the model's default saver\n        directory is used. Optionally appends current timestep to prevent overwriting previous\n        checkpoint files. Turn off to be able to load model from the same given path argument as\n        given here.\n\n        Args:\n            directory (str): Optional checkpoint directory.\n            append_timestep (bool):  Appends the current timestep to the checkpoint file if true.\n                If this is set to True, the load path must include the checkpoint timestep suffix.\n                For example, if stored to models/ and set to true, the exported file will be of the\n                form models/model.ckpt-X where X is the last timestep saved. The load path must\n                precisely match this file name. If this option is turned off, the checkpoint will\n                always overwrite the file specified in path and the model can always be loaded under\n                this path.\n\n        Returns:\n            Checkpoint path were the model was saved.\n        \"\"\"\n        return self.model.save(directory=directory, append_timestep=append_timestep)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef restore_model(self, directory=None, file=None):\n        self.model.restore(directory=directory, file=file)", "response": "Restore TensorFlow model. If no checkpoint file is given, the latest checkpoint is\n        restored. If no checkpoint directory is given, the model's default saver directory is\n        used (unless file specifies the entire path).\n\n        Args:\n            directory: Optional checkpoint directory.\n            file: Optional checkpoint file, or path if directory not given."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates an agent from a specification dict.", "response": "def from_spec(spec, kwargs):\n        \"\"\"\n        Creates an agent from a specification dict.\n        \"\"\"\n        agent = util.get_object(\n            obj=spec,\n            predefined_objects=tensorforce.agents.agents,\n            kwargs=kwargs\n        )\n        assert isinstance(agent, Agent)\n        return agent"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_named_tensor(self, name):\n        if name in self.named_tensors:\n            return True, self.named_tensors[name]\n        else:\n            return False, None", "response": "Returns a named tensor if available."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates a network from a specification dict.", "response": "def from_spec(spec, kwargs=None):\n        \"\"\"\n        Creates a network from a specification dict.\n        \"\"\"\n        network = util.get_object(\n            obj=spec,\n            default_object=LayeredNetwork,\n            kwargs=kwargs\n        )\n        assert isinstance(network, Network)\n        return network"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstore a transition in replay memory.", "response": "def put(self, item, priority=None):\n        \"\"\"\n        Stores a transition in replay memory.\n\n        If the memory is full, the oldest entry is replaced.\n        \"\"\"\n        if not self._isfull():\n            self._memory.append(None)\n        position = self._next_position_then_increment()\n        old_priority = 0 if self._memory[position] is None \\\n            else (self._memory[position].priority or 0)\n        row = _SumRow(item, priority)\n        self._memory[position] = row\n        self._update_internal_nodes(\n            position, (row.priority or 0) - old_priority)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef move(self, external_index, new_priority):\n        index = external_index + (self._capacity - 1)\n        return self._move(index, new_priority)", "response": "Moves the priority of a leaf node."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _move(self, index, new_priority):\n        item, old_priority = self._memory[index]\n        old_priority = old_priority or 0\n        self._memory[index] = _SumRow(item, new_priority)\n        self._update_internal_nodes(index, new_priority - old_priority)", "response": "Move the priority of a leaf node."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nupdating internal nodes in the tree.", "response": "def _update_internal_nodes(self, index, delta):\n        \"\"\"\n        Update internal priority sums when leaf priority has been changed.\n        Args:\n            index: leaf node index\n            delta: change in priority\n        \"\"\"\n        # Move up tree, increasing position, updating sum\n        while index > 0:\n            index = (index - 1) // 2\n            self._memory[index] += delta"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _next_position_then_increment(self):\n        start = self._capacity - 1\n        position = start + self._position\n        self._position = (self._position + 1) % self._capacity\n        return position", "response": "Returns the next available position."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _sample_with_priority(self, p):\n        parent = 0\n        while True:\n            left = 2 * parent + 1\n            if left >= len(self._memory):\n                # parent points to a leaf node already.\n                return parent\n\n            left_p = self._memory[left] if left < self._capacity - 1 \\\n                else (self._memory[left].priority or 0)\n            if p <= left_p:\n                parent = left\n            else:\n                if left + 1 >= len(self._memory):\n                    raise RuntimeError('Right child is expected to exist.')\n                p -= left_p\n                parent = left + 1", "response": "Sample random element with priority greater than p."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsampling the minibatch of size batch_size.", "response": "def sample_minibatch(self, batch_size):\n        \"\"\"\n        Sample minibatch of size batch_size.\n        \"\"\"\n        pool_size = len(self)\n        if pool_size == 0:\n            return []\n\n        delta_p = self._memory[0] / batch_size\n        chosen_idx = []\n        # if all priorities sum to ~0  choose randomly otherwise random sample\n        if abs(self._memory[0]) < util.epsilon:\n            chosen_idx = np.random.randint(self._capacity - 1, self._capacity - 1 + len(self), size=batch_size).tolist()\n        else:\n            for i in xrange(batch_size):\n                lower = max(i * delta_p, 0)\n                upper = min((i + 1) * delta_p, self._memory[0])\n                p = random.uniform(lower, upper)\n                chosen_idx.append(self._sample_with_priority(p))\n        return [(i, self._memory[i]) for i in chosen_idx]"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a batch of the specified size according to priority.", "response": "def get_batch(self, batch_size, next_states=False):\n        \"\"\"\n        Samples a batch of the specified size according to priority.\n\n        Args:\n            batch_size: The batch size\n            next_states: A boolean flag indicating whether 'next_states' values should be included\n\n        Returns: A dict containing states, actions, rewards, terminals, internal states (and next states)\n\n        \"\"\"\n        if batch_size > len(self.observations):\n            raise TensorForceError(\n                \"Requested batch size is larger than observations in memory: increase config.first_update.\")\n\n        # Init empty states\n        states = {name: np.zeros((batch_size,) + tuple(state['shape']), dtype=util.np_dtype(\n            state['type'])) for name, state in self.states_spec.items()}\n        internals = [np.zeros((batch_size,) + shape, dtype)\n                     for shape, dtype in self.internals_spec]\n        actions = {name: np.zeros((batch_size,) + tuple(action['shape']), dtype=util.np_dtype(action['type'])) for name, action in self.actions_spec.items()}\n        terminal = np.zeros((batch_size,), dtype=util.np_dtype('bool'))\n        reward = np.zeros((batch_size,), dtype=util.np_dtype('float'))\n        if next_states:\n            next_states = {name: np.zeros((batch_size,) + tuple(state['shape']), dtype=util.np_dtype(\n                state['type'])) for name, state in self.states_spec.items()}\n            next_internals = [np.zeros((batch_size,) + shape, dtype)\n                              for shape, dtype in self.internals_spec]\n\n        # Start with unseen observations\n        unseen_indices = list(xrange(\n            self.none_priority_index + self.observations._capacity - 1,\n            len(self.observations) + self.observations._capacity - 1)\n        )\n        self.batch_indices = unseen_indices[:batch_size]\n\n        # Get remaining observations using weighted sampling\n        remaining = batch_size - len(self.batch_indices)\n        if remaining:\n            samples = self.observations.sample_minibatch(remaining)\n            sample_indices = [i for i, o in samples]\n            self.batch_indices += sample_indices\n\n        # Shuffle\n        np.random.shuffle(self.batch_indices)\n\n        # Collect observations\n        for n, index in enumerate(self.batch_indices):\n            observation, _ = self.observations._memory[index]\n\n            for name, state in states.items():\n                state[n] = observation[0][name]\n            for k, internal in enumerate(internals):\n                internal[n] = observation[1][k]\n            for name, action in actions.items():\n                action[n] = observation[2][name]\n            terminal[n] = observation[3]\n            reward[n] = observation[4]\n            if next_states:\n                for name, next_state in next_states.items():\n                    next_state[n] = observation[5][name]\n                for k, next_internal in enumerate(next_internals):\n                    next_internal[n] = observation[6][k]\n\n        if next_states:\n            return dict(\n                states=states,\n                internals=internals,\n                actions=actions,\n                terminal=terminal,\n                reward=reward,\n                next_states=next_states,\n                next_internals=next_internals\n            )\n        else:\n            return dict(\n                states=states,\n                internals=internals,\n                actions=actions,\n                terminal=terminal,\n                reward=reward\n            )"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nupdate the internal state of the cluster with the given loss.", "response": "def update_batch(self, loss_per_instance):\n        \"\"\"\n        Computes priorities according to loss.\n\n        Args:\n            loss_per_instance:\n\n        \"\"\"\n        if self.batch_indices is None:\n            raise TensorForceError(\"Need to call get_batch before each update_batch call.\")\n        # if len(loss_per_instance) != len(self.batch_indices):\n        #     raise TensorForceError(\"For all instances a loss value has to be provided.\")\n\n        for index, loss in zip(self.batch_indices, loss_per_instance):\n            # Sampling priority is proportional to the largest absolute temporal difference error.\n            new_priority = (np.abs(loss) + self.prioritization_constant) ** self.prioritization_weight\n            self.observations._move(index, new_priority)\n            self.none_priority_index += 1"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nimport the given list of experiences into the database.", "response": "def import_experience(self, experiences):\n        \"\"\"\n        Imports experiences.\n\n        Args:\n            experiences: \n        \"\"\"\n        if isinstance(experiences, dict):\n            if self.unique_state:\n                experiences['states'] = dict(state=experiences['states'])\n            if self.unique_action:\n                experiences['actions'] = dict(action=experiences['actions'])\n\n            self.model.import_experience(**experiences)\n\n        else:\n            if self.unique_state:\n                states = dict(state=list())\n            else:\n                states = {name: list() for name in experiences[0]['states']}\n            internals = [list() for _ in experiences[0]['internals']]\n            if self.unique_action:\n                actions = dict(action=list())\n            else:\n                actions = {name: list() for name in experiences[0]['actions']}\n            terminal = list()\n            reward = list()\n\n            for experience in experiences:\n                if self.unique_state:\n                    states['state'].append(experience['states'])\n                else:\n                    for name in sorted(states):\n                        states[name].append(experience['states'][name])\n                for n, internal in enumerate(internals):\n                    internal.append(experience['internals'][n])\n                if self.unique_action:\n                    actions['action'].append(experience['actions'])\n                else:\n                    for name in sorted(actions):\n                        actions[name].append(experience['actions'][name])\n                terminal.append(experience['terminal'])\n                reward.append(experience['reward'])\n\n            self.model.import_experience(\n                states=states,\n                internals=internals,\n                actions=actions,\n                terminal=terminal,\n                reward=reward\n            )"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef connect(self, timeout=600):\n        # If we are already connected, return error.\n        if self.socket:\n            raise TensorForceError(\"Already connected to {}:{}. Only one connection allowed at a time. \" +\n                                   \"Close first by calling `close`!\".format(self.host, self.port))\n        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n\n        if timeout < 5 or timeout is None:\n            timeout = 5\n\n        err = 0\n        start_time = time.time()\n        while time.time() - start_time < timeout:\n            self.socket.settimeout(5)\n            err = self.socket.connect_ex((self.host, self.port))\n            if err == 0:\n                break\n            time.sleep(1)\n        if err != 0:\n            raise TensorForceError(\"Error when trying to connect to {}:{}: errno={} errcode='{}' '{}'\".\n                                   format(self.host, self.port, err, errno.errorcode[err], os.strerror(err)))", "response": "Connects to the remote server."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndisconnecting from the server.", "response": "def disconnect(self):\n        \"\"\"\n        Ends our server tcp connection.\n        \"\"\"\n        # If we are not connected, return error.\n        if not self.socket:\n            logging.warning(\"No active socket to close!\")\n            return\n        # Close our socket.\n        self.socket.close()\n        self.socket = None"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef send(self, message, socket_):\n        if not socket_:\n            raise TensorForceError(\"No socket given in call to `send`!\")\n        elif not isinstance(message, dict):\n            raise TensorForceError(\"Message to be sent must be a dict!\")\n        message = msgpack.packb(message)\n        len_ = len(message)\n        # prepend 8-byte len field to all our messages\n        socket_.send(bytes(\"{:08d}\".format(len_), encoding=\"ascii\") + message)", "response": "Sends a message to the socket."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreceiving a message from the given socket and returns it as a dict.", "response": "def recv(self, socket_, encoding=None):\n        \"\"\"\n        Receives a message as msgpack-numpy encoded byte-string from the given socket object.\n        Blocks until something was received.\n\n        Args:\n            socket_: The python socket object to use.\n            encoding (str): The encoding to use for unpacking messages from the socket.\n        Returns: The decoded (as dict) message received.\n        \"\"\"\n        unpacker = msgpack.Unpacker(encoding=encoding)\n\n        # Wait for an immediate response.\n        response = socket_.recv(8)  # get the length of the message\n        if response == b\"\":\n            raise TensorForceError(\"No data received by socket.recv in call to method `recv` \" +\n                                   \"(listener possibly closed)!\")\n        orig_len = int(response)\n        received_len = 0\n        while True:\n            data = socket_.recv(min(orig_len - received_len, self.max_msg_len))\n            # There must be a response.\n            if not data:\n                raise TensorForceError(\"No data of len {} received by socket.recv in call to method `recv`!\".\n                                       format(orig_len - received_len))\n            data_len = len(data)\n            received_len += data_len\n            unpacker.feed(data)\n\n            if received_len == orig_len:\n                break\n\n        # Get the data.\n        for message in unpacker:\n            sts = message.get(\"status\", message.get(b\"status\"))\n            if sts:\n                if sts == \"ok\" or sts == b\"ok\":\n                    return message\n                else:\n                    raise TensorForceError(\"RemoteEnvironment server error: {}\".\n                                           format(message.get(\"message\", \"not specified\")))\n            else:\n                raise TensorForceError(\"Message without field 'status' received!\")\n        raise TensorForceError(\"No message encoded in data stream (data stream had len={})\".\n                               format(orig_len))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef is_action_available(self, action):\n\n        temp_state = np.rot90(self._state, action)\n        return self._is_action_available_left(temp_state)", "response": "Determines whether the action is available."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _is_action_available_left(self, state):\n\n        # True if any field is 0 (empty) on the left of a tile or two tiles can\n        # be merged.\n        for row in range(4):\n            has_empty = False\n            for col in range(4):\n                has_empty |= state[row, col] == 0\n                if state[row, col] != 0 and has_empty:\n                    return True\n                if (state[row, col] != 0 and col > 0 and\n                        state[row, col] == state[row, col - 1]):\n                    return True\n\n        return False", "response": "Determines whether action Left is available."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nexecute action add a new tile update the score & return the reward.", "response": "def do_action(self, action):\n        \"\"\"Execute action, add a new tile, update the score & return the reward.\"\"\"\n\n        temp_state = np.rot90(self._state, action)\n        reward = self._do_action_left(temp_state)\n        self._state = np.rot90(temp_state, -action)\n        self._score += reward\n\n        self.add_random_tile()\n\n        return reward"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding a random tile to the grid. Assumes that the grid has empty fields.", "response": "def add_random_tile(self):\n        \"\"\"Adds a random tile to the grid. Assumes that it has empty fields.\"\"\"\n\n        x_pos, y_pos = np.where(self._state == 0)\n        assert len(x_pos) != 0\n        empty_index = np.random.choice(len(x_pos))\n        value = np.random.choice([1, 2], p=[0.9, 0.1])\n\n        self._state[x_pos[empty_index], y_pos[empty_index]] = value"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef print_state(self):\n\n        def tile_string(value):\n            \"\"\"Concert value to string.\"\"\"\n            if value > 0:\n                return '% 5d' % (2 ** value,)\n            return \"     \"\n\n        separator_line = '-' * 25\n        print(separator_line)\n        for row in range(4):\n            print(\"|\" + \"|\".join([tile_string(v) for v in self._state[row, :]]) + \"|\")\n            print(separator_line)", "response": "Prints the current state of the current user."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset up the TensorFlow model graph and creates the variables and summaries for the current model.", "response": "def setup(self):\n        \"\"\"\n        Sets up the TensorFlow model graph, starts the servers (distributed mode), creates summarizers\n        and savers, initializes (and enters) the TensorFlow session.\n        \"\"\"\n\n        # Create/get our graph, setup local model/global model links, set scope and device.\n        graph_default_context = self.setup_graph()\n\n        # Start a tf Server (in case of distributed setup). Only start once.\n        if self.execution_type == \"distributed\" and self.server is None and self.is_local_model:\n            self.start_server()\n\n        # build the graph\n        with tf.device(device_name_or_function=self.device):\n            with tf.variable_scope(name_or_scope=self.scope, reuse=False):\n\n                # Variables and summaries\n                self.variables = dict()\n                self.all_variables = dict()\n                self.registered_variables = set()\n\n                # Build the graph's placeholders, tf_functions, etc\n                self.setup_placeholders()\n                # Create model's \"external\" components.\n                # Create tensorflow functions from \"tf_\"-methods.\n                self.setup_components_and_tf_funcs()\n\n                # Create core variables (timestep, episode counters, buffers for states/actions/internals).\n                self.fn_initialize()\n\n                if self.summarizer_spec is not None:\n                    with tf.name_scope(name='summarizer'):\n                        self.summarizer = tf.contrib.summary.create_file_writer(\n                            logdir=self.summarizer_spec['directory'],\n                            max_queue=None,\n                            flush_millis=(self.summarizer_spec.get('flush', 10) * 1000),\n                            filename_suffix=None,\n                            name=None\n                        )\n                        default_summarizer = self.summarizer.as_default()\n                        # Problem: not all parts of the graph are called on every step\n                        assert 'steps' not in self.summarizer_spec\n                        # if 'steps' in self.summarizer_spec:\n                        #     record_summaries = tf.contrib.summary.record_summaries_every_n_global_steps(\n                        #         n=self.summarizer_spec['steps'],\n                        #         global_step=self.global_timestep\n                        #     )\n                        # else:\n                        record_summaries = tf.contrib.summary.always_record_summaries()\n\n                    default_summarizer.__enter__()\n                    record_summaries.__enter__()\n\n                # Input tensors\n                states = util.map_tensors(fn=tf.identity, tensors=self.states_input)\n                internals = util.map_tensors(fn=tf.identity, tensors=self.internals_input)\n                actions = util.map_tensors(fn=tf.identity, tensors=self.actions_input)\n                terminal = tf.identity(input=self.terminal_input)\n                reward = tf.identity(input=self.reward_input)\n                # Probably both deterministic and independent should be the same at some point.\n                deterministic = tf.identity(input=self.deterministic_input)\n                independent = tf.identity(input=self.independent_input)\n                episode_index = tf.identity(input=self.episode_index_input)\n\n                states, actions, reward = self.fn_preprocess(states=states, actions=actions, reward=reward)\n\n                self.create_operations(\n                    states=states,\n                    internals=internals,\n                    actions=actions,\n                    terminal=terminal,\n                    reward=reward,\n                    deterministic=deterministic,\n                    independent=independent,\n                    index=episode_index\n                )\n\n                # Add all summaries specified in summary_labels\n                if 'inputs' in self.summary_labels or 'states' in self.summary_labels:\n                    for name in sorted(states):\n                        tf.contrib.summary.histogram(name=('states-' + name), tensor=states[name])\n                if 'inputs' in self.summary_labels or 'actions' in self.summary_labels:\n                    for name in sorted(actions):\n                        tf.contrib.summary.histogram(name=('actions-' + name), tensor=actions[name])\n                if 'inputs' in self.summary_labels or 'reward' in self.summary_labels:\n                    tf.contrib.summary.histogram(name='reward', tensor=reward)\n\n                if 'graph' in self.summary_labels:\n                    with tf.name_scope(name='summarizer'):\n                        graph_def = self.graph.as_graph_def()\n                        graph_str = tf.constant(\n                            value=graph_def.SerializeToString(),\n                            dtype=tf.string,\n                            shape=()\n                        )\n                        self.graph_summary = tf.contrib.summary.graph(\n                            param=graph_str,\n                            step=self.global_timestep\n                        )\n                        if 'meta_param_recorder_class' in self.summarizer_spec:\n                            self.graph_summary = tf.group(\n                                self.graph_summary,\n                                *self.summarizer_spec['meta_param_recorder_class'].build_metagraph_list()\n                            )\n\n                if self.summarizer_spec is not None:\n                    record_summaries.__exit__(None, None, None)\n                    default_summarizer.__exit__(None, None, None)\n\n                    with tf.name_scope(name='summarizer'):\n                        self.flush_summarizer = tf.contrib.summary.flush()\n\n                        self.summarizer_init_op = tf.contrib.summary.summary_writer_initializer_op()\n                        assert len(self.summarizer_init_op) == 1\n                        self.summarizer_init_op = self.summarizer_init_op[0]\n\n        # If we are a global model -> return here.\n        # Saving, syncing, finalizing graph, session is done by local replica model.\n        if self.execution_type == \"distributed\" and not self.is_local_model:\n            return\n\n        # Saver/Summary -> Scaffold.\n        self.setup_saver()\n\n        self.setup_scaffold()\n\n        # Create necessary hooks for the upcoming session.\n        hooks = self.setup_hooks()\n\n        # We are done constructing: Finalize our graph, create and enter the session.\n        self.setup_session(self.server, hooks, graph_default_context)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef setup_graph(self):\n        graph_default_context = None\n\n        # Single (non-distributed) mode.\n        if self.execution_type == \"single\":\n            self.graph = tf.Graph()\n            graph_default_context = self.graph.as_default()\n            graph_default_context.__enter__()\n            self.global_model = None\n\n        # Distributed tf\n        elif self.execution_type == \"distributed\":\n            # Parameter-server -> Do not build any graph.\n            if self.distributed_spec[\"job\"] == \"ps\":\n                return None\n\n            # worker -> construct the global (main) model; the one hosted on the ps,\n            elif self.distributed_spec[\"job\"] == \"worker\":\n                # The local replica model.\n                if self.is_local_model:\n                    graph = tf.Graph()\n                    graph_default_context = graph.as_default()\n                    graph_default_context.__enter__()\n\n                    # Now that the graph is created and entered -> deepcopoy ourselves and setup global model first,\n                    # then continue.\n                    self.global_model = deepcopy(self)\n                    # Switch on global construction/setup-mode for the pass to setup().\n                    self.global_model.is_local_model = False\n                    self.global_model.setup()\n\n                    self.graph = graph\n                    self.as_local_model()\n                    self.scope += '-worker' + str(self.distributed_spec[\"task_index\"])\n                # The global_model (whose Variables are hosted by the ps).\n                else:\n                    self.graph = tf.get_default_graph()  # lives in the same graph as local model\n                    self.global_model = None\n                    self.device = tf.train.replica_device_setter(\n                        # Place its Variables on the parameter server(s) (round robin).\n                        #ps_device=\"/job:ps\",  # default\n                        # Train-ops for the global_model are hosted locally (on this worker's node).\n                        worker_device=self.device,\n                        cluster=self.distributed_spec[\"cluster_spec\"]\n                    )\n            else:\n                raise TensorForceError(\"Unsupported job type: {}!\".format(self.distributed_spec[\"job\"]))\n        else:\n            raise TensorForceError(\"Unsupported distributed type: {}!\".format(self.distributed_spec[\"type\"]))\n\n        return graph_default_context", "response": "Creates our Graph and figures out which global model to hook up to."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef start_server(self):\n        self.server = tf.train.Server(\n            server_or_cluster_def=self.distributed_spec[\"cluster_spec\"],\n            job_name=self.distributed_spec[\"job\"],\n            task_index=self.distributed_spec[\"task_index\"],\n            protocol=self.distributed_spec.get(\"protocol\"),\n            config=self.distributed_spec.get(\"session_config\"),\n            start=True\n        )\n        if self.distributed_spec[\"job\"] == \"ps\":\n            self.server.join()\n            # This is unreachable?\n            quit()", "response": "Creates and stores a tf. train. Server object and optionally joins it if we are a parameter - server."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef setup_placeholders(self):\n\n        # States\n        for name in sorted(self.states_spec):\n            self.states_input[name] = tf.placeholder(\n                dtype=util.tf_dtype(self.states_spec[name]['type']),\n                shape=(None,) + tuple(self.states_spec[name]['shape']),\n                name=('state-' + name)\n            )\n\n        # States preprocessing\n        if self.states_preprocessing_spec is None:\n            for name in sorted(self.states_spec):\n                self.states_spec[name]['unprocessed_shape'] = self.states_spec[name]['shape']\n        elif not isinstance(self.states_preprocessing_spec, list) and \\\n                all(name in self.states_spec for name in self.states_preprocessing_spec):\n            for name in sorted(self.states_spec):\n                if name in self.states_preprocessing_spec:\n                    preprocessing = PreprocessorStack.from_spec(\n                        spec=self.states_preprocessing_spec[name],\n                        kwargs=dict(shape=self.states_spec[name]['shape'])\n                    )\n                    self.states_spec[name]['unprocessed_shape'] = self.states_spec[name]['shape']\n                    self.states_spec[name]['shape'] = preprocessing.processed_shape(shape=self.states_spec[name]['unprocessed_shape'])\n                    self.states_preprocessing[name] = preprocessing\n                else:\n                    self.states_spec[name]['unprocessed_shape'] = self.states_spec[name]['shape']\n        # Single preprocessor for all components of our state space\n        elif \"type\" in self.states_preprocessing_spec:\n            preprocessing = PreprocessorStack.from_spec(spec=self.states_preprocessing_spec,\n                                                        kwargs=dict(shape=self.states_spec[name]['shape']))\n            for name in sorted(self.states_spec):\n                self.states_spec[name]['unprocessed_shape'] = self.states_spec[name]['shape']\n                self.states_spec[name]['shape'] = preprocessing.processed_shape(shape=self.states_spec[name]['unprocessed_shape'])\n                self.states_preprocessing[name] = preprocessing\n        else:\n            for name in sorted(self.states_spec):\n                preprocessing = PreprocessorStack.from_spec(\n                    spec=self.states_preprocessing_spec,\n                    kwargs=dict(shape=self.states_spec[name]['shape'])\n                )\n                self.states_spec[name]['unprocessed_shape'] = self.states_spec[name]['shape']\n                self.states_spec[name]['shape'] = preprocessing.processed_shape(shape=self.states_spec[name]['unprocessed_shape'])\n                self.states_preprocessing[name] = preprocessing\n\n        # Actions\n        for name in sorted(self.actions_spec):\n            self.actions_input[name] = tf.placeholder(\n                dtype=util.tf_dtype(self.actions_spec[name]['type']),\n                shape=(None,) + tuple(self.actions_spec[name]['shape']),\n                name=('action-' + name)\n            )\n\n        # Actions exploration\n        if self.actions_exploration_spec is None:\n            pass\n        elif all(name in self.actions_spec for name in self.actions_exploration_spec):\n            for name in sorted(self.actions_spec):\n                if name in self.actions_exploration:\n                    self.actions_exploration[name] = Exploration.from_spec(spec=self.actions_exploration_spec[name])\n        else:\n            for name in sorted(self.actions_spec):\n                self.actions_exploration[name] = Exploration.from_spec(spec=self.actions_exploration_spec)\n\n        # Terminal\n        self.terminal_input = tf.placeholder(dtype=util.tf_dtype('bool'), shape=(None,), name='terminal')\n\n        # Reward\n        self.reward_input = tf.placeholder(dtype=util.tf_dtype('float'), shape=(None,), name='reward')\n\n        # Reward preprocessing\n        if self.reward_preprocessing_spec is not None:\n            self.reward_preprocessing = PreprocessorStack.from_spec(\n                spec=self.reward_preprocessing_spec,\n                # TODO this can eventually have more complex shapes?\n                kwargs=dict(shape=())\n            )\n            if self.reward_preprocessing.processed_shape(shape=()) != ():\n                raise TensorForceError(\"Invalid reward preprocessing!\")\n\n        # Deterministic/independent action flag (should probably be the same)\n        self.deterministic_input = tf.placeholder(dtype=util.tf_dtype('bool'), shape=(), name='deterministic')\n        self.independent_input = tf.placeholder(dtype=util.tf_dtype('bool'), shape=(), name='independent')", "response": "Creates the TensorFlow placeholders variables ops and functions for the state space."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nallow child models to create model's component objects, such as optimizer(s), memory(s), etc.. Creates all tensorflow functions via tf.make_template calls on all the class' \"tf_\"-methods. Args: custom_getter: The `custom_getter_` object to use for `tf.make_template` when creating TensorFlow functions. If None, use a default custom_getter_. Returns: The custom_getter passed in (or a default one if custom_getter was None).", "response": "def setup_components_and_tf_funcs(self, custom_getter=None):\n        \"\"\"\n        Allows child models to create model's component objects, such as optimizer(s), memory(s), etc..\n        Creates all tensorflow functions via tf.make_template calls on all the class' \"tf_\"-methods.\n\n        Args:\n            custom_getter: The `custom_getter_` object to use for `tf.make_template` when creating TensorFlow functions.\n                If None, use a default custom_getter_.\n\n        Returns: The custom_getter passed in (or a default one if custom_getter was None).\n        \"\"\"\n\n        if custom_getter is None:\n            def custom_getter(getter, name, registered=False, **kwargs):\n                \"\"\"\n                To be passed to tf.make_template() as 'custom_getter_'.\n                \"\"\"\n                if registered:\n                    self.registered_variables.add(name)\n                elif name in self.registered_variables:\n                    registered = True\n                # Top-level, hence no 'registered' argument.\n                variable = getter(name=name, **kwargs)\n                if registered:\n                    pass\n                elif name in self.all_variables:\n                    assert variable is self.all_variables[name]\n                    if kwargs.get('trainable', True):\n                        assert variable is self.variables[name]\n                        if 'variables' in self.summary_labels:\n                            tf.contrib.summary.histogram(name=name, tensor=variable)\n                else:\n                    self.all_variables[name] = variable\n                    if kwargs.get('trainable', True):\n                        self.variables[name] = variable\n                        if 'variables' in self.summary_labels:\n                            tf.contrib.summary.histogram(name=name, tensor=variable)\n                return variable\n\n        self.fn_initialize = tf.make_template(\n            name_='initialize',\n            func_=self.tf_initialize,\n            custom_getter_=custom_getter\n        )\n        self.fn_preprocess = tf.make_template(\n            name_='preprocess',\n            func_=self.tf_preprocess,\n            custom_getter_=custom_getter\n        )\n        self.fn_actions_and_internals = tf.make_template(\n            name_='actions-and-internals',\n            func_=self.tf_actions_and_internals,\n            custom_getter_=custom_getter\n        )\n        self.fn_observe_timestep = tf.make_template(\n            name_='observe-timestep',\n            func_=self.tf_observe_timestep,\n            custom_getter_=custom_getter\n        )\n        self.fn_action_exploration = tf.make_template(\n            name_='action-exploration',\n            func_=self.tf_action_exploration,\n            custom_getter_=custom_getter\n        )\n\n        return custom_getter"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef setup_saver(self):\n        if self.execution_type == \"single\":\n            global_variables = self.get_variables(include_submodules=True, include_nontrainable=True)\n        else:\n            global_variables = self.global_model.get_variables(include_submodules=True, include_nontrainable=True)\n\n        # global_variables += [self.global_episode, self.global_timestep]\n\n        for c in self.get_savable_components():\n            c.register_saver_ops()\n\n        # TensorFlow saver object\n        # TODO potentially make other options configurable via saver spec.\n        self.saver = tf.train.Saver(\n            var_list=global_variables,  # should be given?\n            reshape=False,\n            sharded=False,\n            max_to_keep=5,\n            keep_checkpoint_every_n_hours=10000.0,\n            name=None,\n            restore_sequentially=False,\n            saver_def=None,\n            builder=None,\n            defer_build=False,\n            allow_empty=True,\n            write_version=tf.train.SaverDef.V2,\n            pad_step_number=False,\n            save_relative_paths=True\n            # filename=None\n        )", "response": "Creates the tf. train. Saver object and stores it in self. saver."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates the tf. train. Scaffold object and assigns it to self. scaffold.", "response": "def setup_scaffold(self):\n        \"\"\"\n        Creates the tf.train.Scaffold object and assigns it to self.scaffold.\n        Other fields of the Scaffold are generated automatically.\n        \"\"\"\n        if self.execution_type == \"single\":\n            global_variables = self.get_variables(include_submodules=True, include_nontrainable=True)\n            # global_variables += [self.global_episode, self.global_timestep]\n            init_op = tf.variables_initializer(var_list=global_variables)\n            if self.summarizer_init_op is not None:\n                init_op = tf.group(init_op, self.summarizer_init_op)\n            if self.graph_summary is None:\n                ready_op = tf.report_uninitialized_variables(var_list=global_variables)\n                ready_for_local_init_op = None\n                local_init_op = None\n            else:\n                ready_op = None\n                ready_for_local_init_op = tf.report_uninitialized_variables(var_list=global_variables)\n                local_init_op = self.graph_summary\n\n        else:\n            # Global and local variable initializers.\n            global_variables = self.global_model.get_variables(include_submodules=True, include_nontrainable=True)\n            # global_variables += [self.global_episode, self.global_timestep]\n            local_variables = self.get_variables(include_submodules=True, include_nontrainable=True)\n            init_op = tf.variables_initializer(var_list=global_variables)\n            if self.summarizer_init_op is not None:\n                init_op = tf.group(init_op, self.summarizer_init_op)\n            ready_op = tf.report_uninitialized_variables(var_list=(global_variables + local_variables))\n            ready_for_local_init_op = tf.report_uninitialized_variables(var_list=global_variables)\n            if self.graph_summary is None:\n                local_init_op = tf.group(\n                    tf.variables_initializer(var_list=local_variables),\n                    # Synchronize values of trainable variables.\n                    *(tf.assign(ref=local_var, value=global_var) for local_var, global_var in zip(\n                        self.get_variables(include_submodules=True),\n                        self.global_model.get_variables(include_submodules=True)\n                    ))\n                )\n            else:\n                local_init_op = tf.group(\n                    tf.variables_initializer(var_list=local_variables),\n                    self.graph_summary,\n                    # Synchronize values of trainable variables.\n                    *(tf.assign(ref=local_var, value=global_var) for local_var, global_var in zip(\n                        self.get_variables(include_submodules=True),\n                        self.global_model.get_variables(include_submodules=True)\n                    ))\n                )\n\n        def init_fn(scaffold, session):\n            if self.saver_spec is not None and self.saver_spec.get('load', True):\n                directory = self.saver_spec['directory']\n                file = self.saver_spec.get('file')\n                if file is None:\n                    file = tf.train.latest_checkpoint(\n                        checkpoint_dir=directory,\n                        latest_filename=None  # Corresponds to argument of saver.save() in Model.save().\n                    )\n                elif not os.path.isfile(file):\n                    file = os.path.join(directory, file)\n                if file is not None:\n                    try:\n                        scaffold.saver.restore(sess=session, save_path=file)\n                        session.run(fetches=self.list_buffer_index_reset_op)\n                    except tf.errors.NotFoundError:\n                        raise TensorForceError(\"Error: Existing checkpoint could not be loaded! Set \\\"load\\\" to false in saver_spec.\")\n\n        # TensorFlow scaffold object\n        # TODO explain what it does.\n        self.scaffold = tf.train.Scaffold(\n            init_op=init_op,\n            init_feed_dict=None,\n            init_fn=init_fn,\n            ready_op=ready_op,\n            ready_for_local_init_op=ready_for_local_init_op,\n            local_init_op=local_init_op,\n            summary_op=None,\n            saver=self.saver,\n            copy_from_scaffold=None\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef setup_hooks(self):\n        hooks = list()\n\n        # Checkpoint saver hook\n        if self.saver_spec is not None and (self.execution_type == 'single' or self.distributed_spec['task_index'] == 0):\n            self.saver_directory = self.saver_spec['directory']\n            hooks.append(tf.train.CheckpointSaverHook(\n                checkpoint_dir=self.saver_directory,\n                save_secs=self.saver_spec.get('seconds', None if 'steps' in self.saver_spec else 600),\n                save_steps=self.saver_spec.get('steps'),  # Either one or the other has to be set.\n                saver=None,  # None since given via 'scaffold' argument.\n                checkpoint_basename=self.saver_spec.get('basename', 'model.ckpt'),\n                scaffold=self.scaffold,\n                listeners=None\n            ))\n        else:\n            self.saver_directory = None\n\n        # Stop at step hook\n        # hooks.append(tf.train.StopAtStepHook(\n        #     num_steps=???,  # This makes more sense, if load and continue training.\n        #     last_step=None  # Either one or the other has to be set.\n        # ))\n\n        # # Step counter hook\n        # hooks.append(tf.train.StepCounterHook(\n        #     every_n_steps=counter_config.get('steps', 100),  # Either one or the other has to be set.\n        #     every_n_secs=counter_config.get('secs'),  # Either one or the other has to be set.\n        #     output_dir=None,  # None since given via 'summary_writer' argument.\n        #     summary_writer=summary_writer\n        # ))\n\n        # Other available hooks:\n        # tf.train.FinalOpsHook(final_ops, final_ops_feed_dict=None)\n        # tf.train.GlobalStepWaiterHook(wait_until_step)\n        # tf.train.LoggingTensorHook(tensors, every_n_iter=None, every_n_secs=None)\n        # tf.train.NanTensorHook(loss_tensor, fail_on_nan_loss=True)\n        # tf.train.ProfilerHook(save_steps=None, save_secs=None, output_dir='', show_dataflow=True, show_memory=False)\n\n        return hooks", "response": "Creates and returns a list of hooks to use in a session. Populates self. saver_directory."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates and enters the session for this model.", "response": "def setup_session(self, server, hooks, graph_default_context):\n        \"\"\"\n        Creates and then enters the session for this model (finalizes the graph).\n\n        Args:\n            server (tf.train.Server): The tf.train.Server object to connect to (None for single execution).\n            hooks (list): A list of (saver, summary, etc..) hooks to be passed to the session.\n            graph_default_context: The graph as_default() context that we are currently in.\n        \"\"\"\n        if self.execution_type == \"distributed\":\n            # if self.distributed_spec['task_index'] == 0:\n            # TensorFlow chief session creator object\n            session_creator = tf.train.ChiefSessionCreator(\n                scaffold=self.scaffold,\n                master=server.target,\n                config=self.session_config,\n                checkpoint_dir=None,\n                checkpoint_filename_with_path=None\n            )\n            # else:\n            #     # TensorFlow worker session creator object\n            #     session_creator = tf.train.WorkerSessionCreator(\n            #         scaffold=self.scaffold,\n            #         master=server.target,\n            #         config=self.execution_spec.get('session_config'),\n            #     )\n\n            # TensorFlow monitored session object\n            self.monitored_session = tf.train.MonitoredSession(\n                session_creator=session_creator,\n                hooks=hooks,\n                stop_grace_period_secs=120  # Default value.\n            )\n            # Add debug session.run dumping?\n            if self.tf_session_dump_dir != \"\":\n                self.monitored_session = DumpingDebugWrapperSession(self.monitored_session, self.tf_session_dump_dir)\n        else:\n            # TensorFlow non-distributed monitored session object\n            self.monitored_session = tf.train.SingularMonitoredSession(\n                hooks=hooks,\n                scaffold=self.scaffold,\n                master='',  # Default value.\n                config=self.session_config,  # self.execution_spec.get('session_config'),\n                checkpoint_dir=None\n            )\n\n        if graph_default_context:\n            graph_default_context.__exit__(None, None, None)\n        self.graph.finalize()\n\n        # enter the session to be ready for acting/learning\n        self.monitored_session.__enter__()\n        self.session = self.monitored_session._tf_sess()"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsaves the model (of saver dir is given) and closes the session.", "response": "def close(self):\n        \"\"\"\n        Saves the model (of saver dir is given) and closes the session.\n        \"\"\"\n        if self.flush_summarizer is not None:\n            self.monitored_session.run(fetches=self.flush_summarizer)\n        if self.saver_directory is not None:\n            self.save(append_timestep=True)\n        self.monitored_session.__exit__(None, None, None)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninitializes the variables for the local state and internals and action buffers.", "response": "def tf_initialize(self):\n        \"\"\"\n        Creates tf Variables for the local state/internals/action-buffers and for the local and global counters\n        for timestep and episode.\n        \"\"\"\n\n        # Timesteps/Episodes\n        # Global: (force on global device; local and global model point to the same (global) data).\n        with tf.device(device_name_or_function=(self.global_model.device if self.global_model else self.device)):\n\n            # Global timestep\n            collection = self.graph.get_collection(name='global-timestep')\n            if len(collection) == 0:\n                self.global_timestep = tf.get_variable(\n                    name='global-timestep',\n                    shape=(),\n                    dtype=tf.int64,\n                    trainable=False,\n                    initializer=tf.constant_initializer(value=0, dtype=tf.int64),\n                    collections=['global-timestep', tf.GraphKeys.GLOBAL_STEP]\n                )\n            else:\n                assert len(collection) == 1\n                self.global_timestep = collection[0]\n\n            # Global episode\n            collection = self.graph.get_collection(name='global-episode')\n            if len(collection) == 0:\n                self.global_episode = tf.get_variable(\n                    name='global-episode',\n                    shape=(),\n                    dtype=tf.int64,\n                    trainable=False,\n                    initializer=tf.constant_initializer(value=0, dtype=tf.int64),\n                    collections=['global-episode']\n                )\n            else:\n                assert len(collection) == 1\n                self.global_episode = collection[0]\n\n        # Local counters: local device\n        self.timestep = tf.get_variable(\n            name='timestep',\n            shape=(),\n            dtype=tf.int64,\n            initializer=tf.constant_initializer(value=0, dtype=tf.int64),\n            trainable=False\n        )\n\n        self.episode = tf.get_variable(\n            name='episode',\n            shape=(),\n            dtype=tf.int64,\n            initializer=tf.constant_initializer(value=0, dtype=tf.int64),\n            trainable=False\n        )\n\n        self.episode_index_input = tf.placeholder(\n            name='episode_index',\n            shape=(),\n            dtype=tf.int32,\n        )\n\n        # States buffer variable\n        for name in sorted(self.states_spec):\n            self.list_states_buffer[name] = tf.get_variable(\n                name=('state-{}'.format(name)),\n                shape=((self.num_parallel, self.batching_capacity,) + tuple(self.states_spec[name]['shape'])),\n                dtype=util.tf_dtype(self.states_spec[name]['type']),\n                trainable=False\n            )\n\n        # Internals buffer variable\n        for name in sorted(self.internals_spec):\n            self.list_internals_buffer[name] = tf.get_variable(\n                name=('internal-{}'.format(name)),\n                shape=((self.num_parallel, self.batching_capacity,) + tuple(self.internals_spec[name]['shape'])),\n                dtype=util.tf_dtype(self.internals_spec[name]['type']),\n                trainable=False\n            )\n\n        # Actions buffer variable\n        for name in sorted(self.actions_spec):\n            self.list_actions_buffer[name]= tf.get_variable(\n                name=('action-{}'.format(name)),\n                shape=((self.num_parallel, self.batching_capacity,) + tuple(self.actions_spec[name]['shape'])),\n                dtype=util.tf_dtype(self.actions_spec[name]['type']),\n                trainable=False\n            )\n\n        # Buffer index\n        # for index in range(self.num_parallel):\n        self.list_buffer_index = tf.get_variable(\n            name='buffer-index',\n            shape=(self.num_parallel,),\n            dtype=util.tf_dtype('int'),\n            trainable=False\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\napplies preprocessing ops to the raw states actions and reward tensors.", "response": "def tf_preprocess(self, states, actions, reward):\n        \"\"\"\n        Applies preprocessing ops to the raw states/action/reward inputs.\n\n        Args:\n            states (dict): Dict of raw state tensors.\n            actions (dict): Dict or raw action tensors.\n            reward: 1D (float) raw rewards tensor.\n\n        Returns: The preprocessed versions of the input tensors.\n        \"\"\"\n        # States preprocessing\n        for name in sorted(self.states_preprocessing):\n            states[name] = self.states_preprocessing[name].process(tensor=states[name])\n\n        # Reward preprocessing\n        if self.reward_preprocessing is not None:\n            reward = self.reward_preprocessing.process(tensor=reward)\n\n        return states, actions, reward"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\napplying optional exploration to the action tensor.", "response": "def tf_action_exploration(self, action, exploration, action_spec):\n        \"\"\"\n        Applies optional exploration to the action (post-processor for action outputs).\n\n        Args:\n             action (tf.Tensor): The original output action tensor (to be post-processed).\n             exploration (Exploration): The Exploration object to use.\n             action_spec (dict): Dict specifying the action space.\n        Returns:\n            The post-processed action output tensor.\n        \"\"\"\n        action_shape = tf.shape(input=action)\n        exploration_value = exploration.tf_explore(\n            episode=self.global_episode,\n            timestep=self.global_timestep,\n            shape=action_spec['shape']\n        )\n        exploration_value = tf.expand_dims(input=exploration_value, axis=0)\n\n        if action_spec['type'] == 'bool':\n            action = tf.where(\n                condition=(tf.random_uniform(shape=action_shape) < exploration_value),\n                x=(tf.random_uniform(shape=action_shape) < 0.5),\n                y=action\n            )\n\n        elif action_spec['type'] == 'int':\n            action = tf.where(\n                condition=(tf.random_uniform(shape=action_shape) < exploration_value),\n                x=tf.random_uniform(shape=action_shape, maxval=action_spec['num_actions'], dtype=util.tf_dtype('int')),\n                y=action\n            )\n\n        elif action_spec['type'] == 'float':\n            noise = tf.random_normal(shape=action_shape, dtype=util.tf_dtype('float'))\n            action += noise * exploration_value\n            if 'min_value' in action_spec:\n                action = tf.clip_by_value(\n                    t=action,\n                    clip_value_min=action_spec['min_value'],\n                    clip_value_max=action_spec['max_value']\n                )\n\n        return action"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating and stores tf operations that are fetched when calling act.", "response": "def create_act_operations(self, states, internals, deterministic, independent, index):\n        \"\"\"\n        Creates and stores tf operations that are fetched when calling act(): actions_output, internals_output and\n        timestep_output.\n\n        Args:\n            states (dict): Dict of state tensors (each key represents one state space component).\n            internals (dict): Dict of prior internal state tensors (each key represents one internal state component).\n            deterministic: 0D (bool) tensor (whether to not use action exploration).\n            independent (bool): 0D (bool) tensor (whether to store states/internals/action in local buffer).\n        \"\"\"\n\n        # Optional variable noise\n        operations = list()\n        if self.variable_noise is not None and self.variable_noise > 0.0:\n            # Initialize variables\n            self.fn_actions_and_internals(\n                states=states,\n                internals=internals,\n                deterministic=deterministic\n            )\n\n            noise_deltas = list()\n            for variable in self.get_variables():\n                noise_delta = tf.random_normal(shape=util.shape(variable), mean=0.0, stddev=self.variable_noise)\n                noise_deltas.append(noise_delta)\n                operations.append(variable.assign_add(delta=noise_delta))\n\n        # Retrieve actions and internals\n        with tf.control_dependencies(control_inputs=operations):\n            self.actions_output, self.internals_output = self.fn_actions_and_internals(\n                states=states,\n                internals=internals,\n                deterministic=deterministic\n            )\n\n        # Subtract variable noise\n        # TODO this is an untested/incomplete feature and maybe should be removed for now.\n        with tf.control_dependencies(control_inputs=[self.actions_output[name] for name in sorted(self.actions_output)]):\n            operations = list()\n            if self.variable_noise is not None and self.variable_noise > 0.0:\n                for variable, noise_delta in zip(self.get_variables(), noise_deltas):\n                    operations.append(variable.assign_sub(delta=noise_delta))\n\n        # Actions exploration\n        with tf.control_dependencies(control_inputs=operations):\n            for name in sorted(self.actions_exploration):\n                self.actions_output[name] = tf.cond(\n                    pred=self.deterministic_input,\n                    true_fn=(lambda: self.actions_output[name]),\n                    false_fn=(lambda: self.fn_action_exploration(\n                        action=self.actions_output[name],\n                        exploration=self.actions_exploration[name],\n                        action_spec=self.actions_spec[name]\n                    ))\n                )\n\n        # Independent act not followed by observe.\n        def independent_act():\n            \"\"\"\n            Does not store state, action, internal in buffer. Hence, does not have any influence on learning.\n            Does not increase timesteps.\n            \"\"\"\n            return self.global_timestep\n\n        # Normal act followed by observe, with additional operations.\n        def normal_act():\n            \"\"\"\n            Stores current states, internals and actions in buffer. Increases timesteps.\n            \"\"\"\n            operations = list()\n\n            batch_size = tf.shape(input=states[next(iter(sorted(states)))])[0]\n            for name in sorted(states):\n                operations.append(tf.assign(\n                    ref=self.list_states_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],\n                    value=states[name]\n                ))\n            for name in sorted(internals):\n                operations.append(tf.assign(\n                    ref=self.list_internals_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],\n                    value=internals[name]\n                ))\n            for name in sorted(self.actions_output):\n                operations.append(tf.assign(\n                    ref=self.list_actions_buffer[name][index, self.list_buffer_index[index]: self.list_buffer_index[index] + batch_size],\n                    value=self.actions_output[name]\n                ))\n\n            with tf.control_dependencies(control_inputs=operations):\n                operations = list()\n\n                operations.append(tf.assign(\n                    ref=self.list_buffer_index[index: index+1],\n                    value=tf.add(self.list_buffer_index[index: index+1], tf.constant([1]))\n                ))\n\n                    # Increment timestep\n                operations.append(tf.assign_add(\n                    ref=self.timestep,\n                    value=tf.to_int64(x=batch_size)\n                ))\n                operations.append(tf.assign_add(\n                    ref=self.global_timestep,\n                    value=tf.to_int64(x=batch_size)\n                ))\n\n            with tf.control_dependencies(control_inputs=operations):\n                # Trivial operation to enforce control dependency\n                # TODO why not return no-op?\n                return self.global_timestep + 0\n\n        # Only increment timestep and update buffer if act not independent\n        self.timestep_output = tf.cond(\n            pred=independent,\n            true_fn=independent_act,\n            false_fn=normal_act\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate the operations needed to run an observation batch.", "response": "def create_observe_operations(self, terminal, reward, index):\n        \"\"\"\n        Returns the tf op to fetch when an observation batch is passed in (e.g. an episode's rewards and\n        terminals). Uses the filled tf buffers for states, actions and internals to run\n        the tf_observe_timestep (model-dependent), resets buffer index and increases counters (episodes,\n        timesteps).\n\n        Args:\n            terminal: The 1D tensor (bool) of terminal signals to process (more than one True within that list is ok).\n            reward: The 1D tensor (float) of rewards to process.\n\n        Returns: Tf op to fetch when `observe()` is called.\n        \"\"\"\n        # Increment episode\n        num_episodes = tf.count_nonzero(input_tensor=terminal, dtype=util.tf_dtype('int'))\n        increment_episode = tf.assign_add(ref=self.episode, value=tf.to_int64(x=num_episodes))\n        increment_global_episode = tf.assign_add(ref=self.global_episode, value=tf.to_int64(x=num_episodes))\n\n        with tf.control_dependencies(control_inputs=(increment_episode, increment_global_episode)):\n            # Stop gradients\n            fn = (lambda x: tf.stop_gradient(input=x[:self.list_buffer_index[index]]))\n            states = util.map_tensors(fn=fn, tensors=self.list_states_buffer, index=index)\n            internals = util.map_tensors(fn=fn, tensors=self.list_internals_buffer, index=index)\n            actions = util.map_tensors(fn=fn, tensors=self.list_actions_buffer, index=index)\n            terminal = tf.stop_gradient(input=terminal)\n            reward = tf.stop_gradient(input=reward)\n\n            # Observation\n            observation = self.fn_observe_timestep(\n                states=states,\n                internals=internals,\n                actions=actions,\n                terminal=terminal,\n                reward=reward\n            )\n\n        with tf.control_dependencies(control_inputs=(observation,)):\n            # Reset buffer index.\n            reset_index = tf.assign(ref=self.list_buffer_index[index], value=0)\n\n        with tf.control_dependencies(control_inputs=(reset_index,)):\n            # Trivial operation to enforce control dependency.\n            self.episode_output = self.global_episode + 0\n\n        self.list_buffer_index_reset_op = tf.group(\n            *(tf.assign(ref=self.list_buffer_index[n], value=0) for n in range(self.num_parallel))\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_atomic_observe_operations(self, states, actions, internals, terminal, reward, index):\n        # Increment episode\n        num_episodes = tf.count_nonzero(input_tensor=terminal, dtype=util.tf_dtype('int'))\n        increment_episode = tf.assign_add(ref=self.episode, value=tf.to_int64(x=num_episodes))\n        increment_global_episode = tf.assign_add(ref=self.global_episode, value=tf.to_int64(x=num_episodes))\n\n        with tf.control_dependencies(control_inputs=(increment_episode, increment_global_episode)):\n            # Stop gradients\n            # Not using buffers here.\n            states = util.map_tensors(fn=tf.stop_gradient, tensors=states)\n            internals = util.map_tensors(fn=tf.stop_gradient, tensors=internals)\n            actions = util.map_tensors(fn=tf.stop_gradient, tensors=actions)\n            terminal = tf.stop_gradient(input=terminal)\n            reward = tf.stop_gradient(input=reward)\n\n            # Observation\n            observation = self.fn_observe_timestep(\n                states=states,\n                internals=internals,\n                actions=actions,\n                terminal=terminal,\n                reward=reward\n            )\n\n        with tf.control_dependencies(control_inputs=(observation,)):\n            # Trivial operation to enforce control dependency.\n            self.unbuffered_episode_output = self.global_episode + 0", "response": "Creates the atomic operations needed to fetch unbuffered observations."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_operations(self, states, internals, actions, terminal, reward, deterministic, independent, index):\n        self.create_act_operations(\n            states=states,\n            internals=internals,\n            deterministic=deterministic,\n            independent=independent,\n            index=index\n        )\n        self.create_observe_operations(\n            reward=reward,\n            terminal=terminal,\n            index=index\n        )\n        self.create_atomic_observe_operations(\n            states=states,\n            actions=actions,\n            internals=internals,\n            reward=reward,\n            terminal=terminal,\n            index=index\n        )", "response": "Creates and stores tf operations for when act and observe are called."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the TensorFlow variables used by the model.", "response": "def get_variables(self, include_submodules=False, include_nontrainable=False):\n        \"\"\"\n        Returns the TensorFlow variables used by the model.\n\n        Args:\n            include_submodules: Includes variables of submodules (e.g. baseline, target network)\n                if true.\n            include_nontrainable: Includes non-trainable variables if true.\n\n        Returns:\n            List of variables.\n        \"\"\"\n        if include_nontrainable:\n            model_variables = [self.all_variables[key] for key in sorted(self.all_variables)]\n\n            states_preprocessing_variables = [\n                variable for name in sorted(self.states_preprocessing)\n                for variable in self.states_preprocessing[name].get_variables()\n            ]\n            model_variables += states_preprocessing_variables\n\n            actions_exploration_variables = [\n                variable for name in sorted(self.actions_exploration)\n                for variable in self.actions_exploration[name].get_variables()\n            ]\n            model_variables += actions_exploration_variables\n\n            if self.reward_preprocessing is not None:\n                reward_preprocessing_variables = self.reward_preprocessing.get_variables()\n                model_variables += reward_preprocessing_variables\n\n        else:\n            model_variables = [self.variables[key] for key in sorted(self.variables)]\n\n        return model_variables"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef reset(self):\n        fetches = [self.global_episode, self.global_timestep]\n\n        # Loop through all preprocessors and reset them as well.\n        for name in sorted(self.states_preprocessing):\n            fetch = self.states_preprocessing[name].reset()\n            if fetch is not None:\n                fetches.extend(fetch)\n\n        if self.flush_summarizer is not None:\n            fetches.append(self.flush_summarizer)\n\n        # Get the updated episode and timestep counts.\n        fetch_list = self.monitored_session.run(fetches=fetches)\n        episode, timestep = fetch_list[:2]\n\n        return episode, timestep, self.internals_init", "response": "Resets the model to its initial state on episode start. This should also reset all preprocessor and state."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the feed - dict for the model s acting and observing tf fetches.", "response": "def get_feed_dict(\n        self,\n        states=None,\n        internals=None,\n        actions=None,\n        terminal=None,\n        reward=None,\n        deterministic=None,\n        independent=None,\n        index=None\n    ):\n        \"\"\"\n        Returns the feed-dict for the model's acting and observing tf fetches.\n\n        Args:\n            states (dict): Dict of state values (each key represents one state space component).\n            internals (dict): Dict of internal state values (each key represents one internal state component).\n            actions (dict): Dict of actions (each key represents one action space component).\n            terminal (List[bool]): List of is-terminal signals.\n            reward (List[float]): List of reward signals.\n            deterministic (bool): Whether actions should be picked without exploration.\n            independent (bool): Whether we are doing an independent act (not followed by call to observe;\n                not to be stored in model's buffer).\n\n        Returns: The feed dict to use for the fetch.\n        \"\"\"\n        feed_dict = dict()\n        batched = None\n\n        if states is not None:\n            if batched is None:\n                name = next(iter(states))\n                state = np.asarray(states[name])\n                batched = (state.ndim != len(self.states_spec[name]['unprocessed_shape']))\n            if batched:\n                feed_dict.update({self.states_input[name]: states[name] for name in sorted(self.states_input)})\n            else:\n                feed_dict.update({self.states_input[name]: (states[name],) for name in sorted(self.states_input)})\n\n        if internals is not None:\n            if batched is None:\n                name = next(iter(internals))\n                internal = np.asarray(internals[name])\n                batched = (internal.ndim != len(self.internals_spec[name]['shape']))\n            if batched:\n                feed_dict.update({self.internals_input[name]: internals[name] for name in sorted(self.internals_input)})\n            else:\n                feed_dict.update({self.internals_input[name]: (internals[name],) for name in sorted(self.internals_input)})\n\n        if actions is not None:\n            if batched is None:\n                name = next(iter(actions))\n                action = np.asarray(actions[name])\n                batched = (action.ndim != len(self.actions_spec[name]['shape']))\n            if batched:\n                feed_dict.update({self.actions_input[name]: actions[name] for name in sorted(self.actions_input)})\n            else:\n                feed_dict.update({self.actions_input[name]: (actions[name],) for name in sorted(self.actions_input)})\n\n        if terminal is not None:\n            if batched is None:\n                terminal = np.asarray(terminal)\n                batched = (terminal.ndim == 1)\n            if batched:\n                feed_dict[self.terminal_input] = terminal\n            else:\n                feed_dict[self.terminal_input] = (terminal,)\n\n        if reward is not None:\n            if batched is None:\n                reward = np.asarray(reward)\n                batched = (reward.ndim == 1)\n            if batched:\n                feed_dict[self.reward_input] = reward\n            else:\n                feed_dict[self.reward_input] = (reward,)\n\n        if deterministic is not None:\n            feed_dict[self.deterministic_input] = deterministic\n\n        if independent is not None:\n            feed_dict[self.independent_input] = independent\n\n        feed_dict[self.episode_index_input] = index\n\n        return feed_dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndo a forward pass through the model to retrieve action (outputs) given inputs for state (and internal state, if applicable (e.g. RNNs)) Args: states (dict): Dict of state values (each key represents one state space component). internals (dict): Dict of internal state values (each key represents one internal state component). deterministic (bool): If True, will not apply exploration after actions are calculated. independent (bool): If true, action is not followed by observe (and hence not included in updates). fetch_tensors (list): List of names of additional tensors (from the model's network) to fetch (and return). index: (int) index of the episode we want to produce the next action Returns: tuple: - Actual action-outputs (batched if state input is a batch). - Actual values of internal states (if applicable) (batched if state input is a batch). - The timestep (int) after calculating the (batch of) action(s).", "response": "def act(self, states, internals, deterministic=False, independent=False, fetch_tensors=None, index=0):\n        \"\"\"\n        Does a forward pass through the model to retrieve action (outputs) given inputs for state (and internal\n        state, if applicable (e.g. RNNs))\n\n        Args:\n            states (dict): Dict of state values (each key represents one state space component).\n            internals (dict): Dict of internal state values (each key represents one internal state component).\n            deterministic (bool): If True, will not apply exploration after actions are calculated.\n            independent (bool): If true, action is not followed by observe (and hence not included\n                in updates).\n            fetch_tensors (list): List of names of additional tensors (from the model's network) to fetch (and return).\n            index: (int) index of the episode we want to produce the next action\n\n        Returns:\n            tuple:\n                - Actual action-outputs (batched if state input is a batch).\n                - Actual values of internal states (if applicable) (batched if state input is a batch).\n                - The timestep (int) after calculating the (batch of) action(s).\n        \"\"\"\n        name = next(iter(states))\n        state = np.asarray(states[name])\n        batched = (state.ndim != len(self.states_spec[name]['unprocessed_shape']))\n        if batched:\n            assert state.shape[0] <= self.batching_capacity\n\n        fetches = [self.actions_output, self.internals_output, self.timestep_output]\n        if self.network is not None and fetch_tensors is not None:\n            for name in fetch_tensors:\n                valid, tensor = self.network.get_named_tensor(name)\n                if valid:\n                    fetches.append(tensor)\n                else:\n                    keys = self.network.get_list_of_named_tensor()\n                    raise TensorForceError('Cannot fetch named tensor \"{}\", Available {}.'.format(name, keys))\n\n        # feed_dict[self.deterministic_input] = deterministic\n        feed_dict = self.get_feed_dict(\n            states=states,\n            internals=internals,\n            deterministic=deterministic,\n            independent=independent,\n            index=index\n        )\n\n        fetch_list = self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)\n        actions, internals, timestep = fetch_list[0:3]\n\n        # Extract the first (and only) action/internal from the batch to make return values non-batched\n        if not batched:\n            actions = {name: actions[name][0] for name in sorted(actions)}\n            internals = {name: internals[name][0] for name in sorted(internals)}\n\n        if self.network is not None and fetch_tensors is not None:\n            fetch_dict = dict()\n            for index_, tensor in enumerate(fetch_list[3:]):\n                name = fetch_tensors[index_]\n                fetch_dict[name] = tensor\n            return actions, internals, timestep, fetch_dict\n        else:\n            return actions, internals, timestep"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding an observation to the model without updating its trainable variables.", "response": "def observe(self, terminal, reward, index=0):\n        \"\"\"\n        Adds an observation (reward and is-terminal) to the model without updating its trainable variables.\n\n        Args:\n            terminal (List[bool]): List of is-terminal signals.\n            reward (List[float]): List of reward signals.\n            index: (int) parallel episode you want to observe\n\n        Returns:\n            The value of the model-internal episode counter.\n        \"\"\"\n        fetches = self.episode_output\n        feed_dict = self.get_feed_dict(terminal=terminal, reward=reward, index=index)\n\n        episode = self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)\n\n        return episode"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsaves the current state of the current model to a checkpoint file.", "response": "def save(self, directory=None, append_timestep=True):\n        \"\"\"\n        Save TensorFlow model. If no checkpoint directory is given, the model's default saver\n        directory is used. Optionally appends current timestep to prevent overwriting previous\n        checkpoint files. Turn off to be able to load model from the same given path argument as\n        given here.\n\n        Args:\n            directory: Optional checkpoint directory.\n            append_timestep: Appends the current timestep to the checkpoint file if true.\n\n        Returns:\n            Checkpoint path where the model was saved.\n        \"\"\"\n        if self.flush_summarizer is not None:\n            self.monitored_session.run(fetches=self.flush_summarizer)\n\n        return self.saver.save(\n            sess=self.session,\n            save_path=(self.saver_directory if directory is None else directory),\n            global_step=(self.global_timestep if append_timestep else None),\n            # latest_filename=None,  # Defaults to 'checkpoint'.\n            meta_graph_suffix='meta',\n            write_meta_graph=True,\n            write_state=True\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef restore(self, directory=None, file=None):\n        if file is None:\n            file = tf.train.latest_checkpoint(\n                checkpoint_dir=(self.saver_directory if directory is None else directory),\n                # latest_filename=None  # Corresponds to argument of saver.save() in Model.save().\n            )\n        elif directory is None:\n            file = os.path.join(self.saver_directory, file)\n        elif not os.path.isfile(file):\n            file = os.path.join(directory, file)\n\n        # if not os.path.isfile(file):\n        #     raise TensorForceError(\"Invalid model directory/file.\")\n\n        self.saver.restore(sess=self.session, save_path=file)\n        self.session.run(fetches=self.list_buffer_index_reset_op)", "response": "Restore TensorFlow model. If no checkpoint file is given, the latest checkpoint is\n        restored. If no checkpoint directory is given, the model's default saver directory is\n        used (unless file specifies the entire path).\n\n        Args:\n            directory: Optional checkpoint directory.\n            file: Optional checkpoint file, or path if directory not given."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the list of all of the components that can be individually saved and restored.", "response": "def get_savable_components(self):\n        \"\"\"\n        Returns the list of all of the components this model consists of that can be individually saved and restored.\n        For instance the network or distribution.\n\n        Returns:\n            List of util.SavableComponent\n        \"\"\"\n        components = self.get_components()\n        components = [components[name] for name in sorted(components)]\n        return set(filter(lambda x: isinstance(x, util.SavableComponent), components))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef save_component(self, component_name, save_path):\n        component = self.get_component(component_name=component_name)\n        self._validate_savable(component=component, component_name=component_name)\n        return component.save(sess=self.session, save_path=save_path)", "response": "Saves a component of this model to the designated location."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef restore_component(self, component_name, save_path):\n        component = self.get_component(component_name=component_name)\n        self._validate_savable(component=component, component_name=component_name)\n        component.restore(sess=self.session, save_path=save_path)", "response": "Restores a component s parameters from a save location."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the component with the provided name or None if there is no such component.", "response": "def get_component(self, component_name):\n        \"\"\"\n        Looks up a component by its name.\n\n        Args:\n            component_name: The name of the component to look up.\n        Returns:\n            The component for the provided name or None if there is no such component.\n        \"\"\"\n        mapping = self.get_components()\n        return mapping[component_name] if component_name in mapping else None"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nimports the demonstrations in the database into memory.", "response": "def import_demonstrations(self, demonstrations):\n        \"\"\"\n        Imports demonstrations, i.e. expert observations. Note that for large numbers of observations,\n        set_demonstrations is more appropriate, which directly sets memory contents to an array an expects\n        a different layout.\n\n        Args:\n            demonstrations: List of observation dicts\n        \"\"\"\n        if isinstance(demonstrations, dict):\n            if self.unique_state:\n                demonstrations['states'] = dict(state=demonstrations['states'])\n            if self.unique_action:\n                demonstrations['actions'] = dict(action=demonstrations['actions'])\n\n            self.model.import_demo_experience(**demonstrations)\n\n        else:\n            if self.unique_state:\n                states = dict(state=list())\n            else:\n                states = {name: list() for name in demonstrations[0]['states']}\n            internals = {name: list() for name in demonstrations[0]['internals']}\n            if self.unique_action:\n                actions = dict(action=list())\n            else:\n                actions = {name: list() for name in demonstrations[0]['actions']}\n            terminal = list()\n            reward = list()\n\n            for demonstration in demonstrations:\n                if self.unique_state:\n                    states['state'].append(demonstration['states'])\n                else:\n                    for name, state in states.items():\n                        state.append(demonstration['states'][name])\n                for name, internal in internals.items():\n                    internal.append(demonstration['internals'][name])\n                if self.unique_action:\n                    actions['action'].append(demonstration['actions'])\n                else:\n                    for name, action in actions.items():\n                        action.append(demonstration['actions'][name])\n                terminal.append(demonstration['terminal'])\n                reward.append(demonstration['reward'])\n\n            self.model.import_demo_experience(\n                states=states,\n                internals=internals,\n                actions=actions,\n                terminal=terminal,\n                reward=reward\n            )"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef seed(self, seed): # pylint: disable=E0202\n        if seed is None:\n            self.env.seed = round(time.time())\n        else:\n            self.env.seed = seed\n        return self.env.seed", "response": "Sets the random seed of the environment."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nexecuting action and returns next state done and reward.", "response": "def execute(self, action):\n        \"\"\"\n        Executes action, observes next state and reward.\n\n        Args:\n            actions: Action to execute.\n\n        Returns:\n            (Dict of) next state(s), boolean indicating terminal, and reward signal.\n        \"\"\"\n        if self.env.game_over():\n            return self.env.getScreenRGB(), True, 0\n\n        action_space = self.env.getActionSet()\n        reward = self.env.act(action_space[action])\n        new_state = self.env.getScreenRGB()\n        done = self.env.game_over()\n        return new_state, done, reward"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the state space. Might include subdicts.", "response": "def states(self):\n        \"\"\"\n        Return the state space. Might include subdicts if multiple states are\n        available simultaneously.\n\n        Returns: dict of state properties (shape and type).\n\n        \"\"\"\n        screen = self.env.getScreenRGB()\n        return dict(shape=screen.shape, type='int')"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef sanity_check_actions(actions_spec):\n    # Leave incoming spec-dict intact.\n    actions = copy.deepcopy(actions_spec)\n\n    # Unique action shortform.\n    is_unique = ('type' in actions)\n    if is_unique:\n        actions = dict(action=actions)\n\n    # Normalize actions.\n    for name, action in actions.items():\n        # Set default type to int\n        if 'type' not in action:\n            action['type'] = 'int'\n\n        # Check required values\n        if action['type'] == 'int':\n            if 'num_actions' not in action:\n                raise TensorForceError(\"Action requires value 'num_actions' set!\")\n        elif action['type'] == 'float':\n            if ('min_value' in action) != ('max_value' in action):\n                raise TensorForceError(\"Action requires both values 'min_value' and 'max_value' set!\")\n\n        # Set default shape to empty tuple (single-int, discrete action space)\n        if 'shape' not in action:\n            action['shape'] = ()\n\n        # Convert int to unary tuple\n        if isinstance(action['shape'], int):\n            action['shape'] = (action['shape'],)\n\n    return actions, is_unique", "response": "Sanity checks an actions dict for MDP."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef sanity_check_execution_spec(execution_spec):\n\n    # default spec: single mode\n    def_ = dict(type=\"single\",\n                distributed_spec=None,\n                session_config=None)\n\n    if execution_spec is None:\n        return def_\n\n    assert isinstance(execution_spec, dict), \"ERROR: execution-spec needs to be of type dict (but is of type {})!\".\\\n        format(type(execution_spec).__name__)\n\n    type_ = execution_spec.get(\"type\")\n\n    # TODO: Figure out what exactly we need for options and what types we should support.\n    if type_ == \"distributed\":\n        def_ = dict(job=\"ps\", task_index=0, cluster_spec={\n            \"ps\": [\"localhost:22222\"],\n            \"worker\": [\"localhost:22223\"]\n        })\n        def_.update(execution_spec.get(\"distributed_spec\", {}))\n        execution_spec[\"distributed_spec\"] = def_\n        execution_spec[\"session_config\"] = execution_spec.get(\"session_config\")\n        return execution_spec\n    elif type_ == \"multi-threaded\":\n        return execution_spec\n    elif type_ == \"single\":\n        return execution_spec\n\n    if execution_spec.get('num_parallel') != None:\n        assert type(execution_spec['num_parallel']) is int, \"ERROR: num_parallel needs to be of type int but is of type {}!\".format(type(execution_spec['num_parallel']).__name__)\n        assert execution_spec['num_parallel'] > 0, \"ERROR: num_parallel needs to be > 0 but is equal to {}\".format(execution_spec['num_parallel'])\n        return execution_spec\n\n    raise TensorForceError(\"Unsupported execution type specified ({})!\".format(type_))", "response": "Sanity checks a execution - spec dict."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nbuild and returns an Extraterrestrial Marauders game.", "response": "def make_game():\n  \"\"\"Builds and returns an Extraterrestrial Marauders game.\"\"\"\n  return ascii_art.ascii_art_to_game(\n      GAME_ART, what_lies_beneath=' ',\n      sprites=dict(\n          [('P', PlayerSprite)] +\n          [(c, UpwardLaserBoltSprite) for c in UPWARD_BOLT_CHARS] +\n          [(c, DownwardLaserBoltSprite) for c in DOWNWARD_BOLT_CHARS]),\n      drapes=dict(X=MarauderDrape,\n                  B=BunkerDrape),\n      update_schedule=['P', 'B', 'X'] + list(_ALL_BOLT_CHARS))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _fly(self, board, layers, things, the_plot):\n    # Disappear if we've hit a Marauder or a bunker.\n    if (self.character in the_plot['bunker_hitters'] or\n        self.character in the_plot['marauder_hitters']):\n      return self._teleport((-1, -1))\n    # Otherwise, northward!\n    self._north(board, the_plot)", "response": "Handles the behaviour of visible bolts flying toward Marauders."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _fire(self, layers, things, the_plot):\n    # We don't fire if the player fired another bolt just now.\n    if the_plot.get('last_player_shot') == the_plot.frame: return\n    the_plot['last_player_shot'] = the_plot.frame\n    # We start just above the player.\n    row, col = things['P'].position\n    self._teleport((row-1, col))", "response": "Launches a new bolt from the player."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _fly(self, board, layers, things, the_plot):\n    # Disappear if we've hit a bunker.\n    if self.character in the_plot['bunker_hitters']:\n      return self._teleport((-1, -1))\n    # End the game if we've hit the player.\n    if self.position == things['P'].position: the_plot.terminate_episode()\n    self._south(board, the_plot)", "response": "Handles the behaviour of visible bolts flying toward the player."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nlaunch a new bolt from a random Marauder.", "response": "def _fire(self, layers, the_plot):\n    \"\"\"Launches a new bolt from a random Marauder.\"\"\"\n    # We don't fire if another Marauder fired a bolt just now.\n    if the_plot.get('last_marauder_shot') == the_plot.frame: return\n    the_plot['last_marauder_shot'] = the_plot.frame\n    # Which Marauder should fire the laser bolt?\n    col = np.random.choice(np.nonzero(layers['X'].sum(axis=0))[0])\n    row = np.nonzero(layers['X'][:, col])[0][-1] + 1\n    # Move ourselves just below that Marauder.\n    self._teleport((row, col))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nresetting the Runner s internal stats counters.", "response": "def reset(self, history=None):\n        \"\"\"\n        Resets the Runner's internal stats counters.\n        If history is empty, use default values in history.get().\n\n        Args:\n            history (dict): A dictionary containing an already run experiment's results. Keys should be:\n                episode_rewards (list of rewards), episode_timesteps (lengths of episodes), episode_times (run-times)\n        \"\"\"\n        if not history:\n            history = dict()\n\n        self.episode_rewards = history.get(\"episode_rewards\", list())\n        self.episode_timesteps = history.get(\"episode_timesteps\", list())\n        self.episode_times = history.get(\"episode_times\", list())"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nexecutes this runner by starting to act (via Agent(s)) in the given Environment(s). Stops execution according to certain conditions (e.g. max. number of episodes, etc..). Calls callback functions after each episode and/or after some summary criteria are met. Args: num_episodes (int): Max. number of episodes to run globally in total (across all threads/workers). num_timesteps (int): Max. number of time steps to run globally in total (across all threads/workers) max_episode_timesteps (int): Max. number of timesteps per episode. deterministic (bool): Whether to use exploration when selecting actions. episode_finished (callable): A function to be called once an episodes has finished. Should take a BaseRunner object and some worker ID (e.g. thread-ID or task-ID). Can decide for itself every how many episodes it should report something and what to report. summary_report (callable): Deprecated; Function that could produce a summary over the training progress so far. summary_interval (int): Deprecated; The number of time steps to execute (globally) before summary_report is called.", "response": "def run(self, num_episodes, num_timesteps, max_episode_timesteps, deterministic, episode_finished, summary_report,\n            summary_interval):\n        \"\"\"\n        Executes this runner by starting to act (via Agent(s)) in the given Environment(s).\n        Stops execution according to certain conditions (e.g. max. number of episodes, etc..).\n        Calls callback functions after each episode and/or after some summary criteria are met.\n\n        Args:\n            num_episodes (int): Max. number of episodes to run globally in total (across all threads/workers).\n            num_timesteps (int): Max. number of time steps to run globally in total (across all threads/workers)\n            max_episode_timesteps (int): Max. number of timesteps per episode.\n            deterministic (bool): Whether to use exploration when selecting actions.\n            episode_finished (callable): A function to be called once an episodes has finished. Should take\n                a BaseRunner object and some worker ID (e.g. thread-ID or task-ID). Can decide for itself\n                every how many episodes it should report something and what to report.\n            summary_report (callable): Deprecated; Function that could produce a summary over the training\n                progress so far.\n            summary_interval (int): Deprecated; The number of time steps to execute (globally)\n                before summary_report is called.\n        \"\"\"\n        raise NotImplementedError"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nretrieve the experiences for given indices by combining entries from buffer elements which have no priorities and entries from priority memory.", "response": "def tf_retrieve_indices(self, buffer_elements, priority_indices):\n        \"\"\"\n        Fetches experiences for given indices by combining entries from buffer\n        which have no priorities, and entries from priority memory.\n\n        Args:\n            buffer_elements: Number of buffer elements to retrieve\n            priority_indices: Index tensor for priority memory\n\n        Returns: Batch of experiences\n        \"\"\"\n        states = dict()\n        buffer_start = self.buffer_index - buffer_elements\n        buffer_end = self.buffer_index\n\n        # Fetch entries from respective memories, concat.\n        for name in sorted(self.states_memory):\n            buffer_state_memory = self.states_buffer[name]\n            # Slicing is more efficient than gathering, and buffer elements are always\n            # fetched using contiguous indices.\n            buffer_states = buffer_state_memory[buffer_start:buffer_end]\n            # Memory indices are obtained via priority sampling, hence require gather.\n            memory_states = tf.gather(params=self.states_memory[name], indices=priority_indices)\n            states[name] = tf.concat(values=(buffer_states, memory_states), axis=0)\n\n        internals = dict()\n        for name in sorted(self.internals_memory):\n            internal_buffer_memory = self.internals_buffer[name]\n            buffer_internals = internal_buffer_memory[buffer_start:buffer_end]\n            memory_internals = tf.gather(params=self.internals_memory[name], indices=priority_indices)\n            internals[name] = tf.concat(values=(buffer_internals, memory_internals), axis=0)\n\n        actions = dict()\n        for name in sorted(self.actions_memory):\n            action_buffer_memory = self.actions_buffer[name]\n            buffer_action = action_buffer_memory[buffer_start:buffer_end]\n            memory_action = tf.gather(params=self.actions_memory[name], indices=priority_indices)\n            actions[name] = tf.concat(values=(buffer_action, memory_action), axis=0)\n\n        buffer_terminal = self.terminal_buffer[buffer_start:buffer_end]\n        priority_terminal = tf.gather(params=self.terminal_memory, indices=priority_indices)\n        terminal = tf.concat(values=(buffer_terminal, priority_terminal), axis=0)\n\n        buffer_reward = self.reward_buffer[buffer_start:buffer_end]\n        priority_reward = tf.gather(params=self.reward_memory, indices=priority_indices)\n        reward = tf.concat(values=(buffer_reward, priority_reward), axis=0)\n\n        if self.include_next_states:\n            assert util.rank(priority_indices) == 1\n            next_priority_indices = (priority_indices + 1) % self.capacity\n            next_buffer_start = (buffer_start + 1) % self.buffer_size\n            next_buffer_end = (buffer_end + 1) % self.buffer_size\n\n            next_states = dict()\n            for name in sorted(self.states_memory):\n                buffer_state_memory = self.states_buffer[name]\n                buffer_next_states = buffer_state_memory[next_buffer_start:next_buffer_end]\n                memory_next_states = tf.gather(params=self.states_memory[name], indices=next_priority_indices)\n                next_states[name] = tf.concat(values=(buffer_next_states, memory_next_states), axis=0)\n\n            next_internals = dict()\n            for name in sorted(self.internals_memory):\n                buffer_internal_memory = self.internals_buffer[name]\n                buffer_next_internals = buffer_internal_memory[next_buffer_start:next_buffer_end]\n                memory_next_internals = tf.gather(params=self.internals_memory[name], indices=next_priority_indices)\n                next_internals[name] = tf.concat(values=(buffer_next_internals, memory_next_internals), axis=0)\n\n            return dict(\n                states=states,\n                internals=internals,\n                actions=actions,\n                terminal=terminal,\n                reward=reward,\n                next_states=next_states,\n                next_internals=next_internals\n            )\n        else:\n            return dict(\n                states=states,\n                internals=internals,\n                actions=actions,\n                terminal=terminal,\n                reward=reward\n            )"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef tf_update_batch(self, loss_per_instance):\n        # 1. We reconstruct the batch from the buffer and the priority memory via\n        # the TensorFlow variables holding the respective indices.\n        mask = tf.not_equal(\n            x=self.batch_indices,\n            y=tf.zeros(shape=tf.shape(input=self.batch_indices), dtype=tf.int32)\n        )\n        priority_indices = tf.reshape(tensor=tf.where(condition=mask), shape=[-1])\n\n        # These are elements from the buffer which first need to be inserted into the main memory.\n        sampled_buffer_batch = self.tf_retrieve_indices(\n            buffer_elements=self.last_batch_buffer_elems,\n            priority_indices=priority_indices\n        )\n\n        # Extract batch elements.\n        states = sampled_buffer_batch['states']\n        internals = sampled_buffer_batch['internals']\n        actions = sampled_buffer_batch['actions']\n        terminal = sampled_buffer_batch['terminal']\n        reward = sampled_buffer_batch['reward']\n\n        # 2. Compute priorities for all batch elements.\n        priorities = loss_per_instance ** self.prioritization_weight\n        assignments = list()\n\n        # 3. Insert the buffer elements from the recent batch into memory,\n        # overwrite memory if full.\n        memory_end_index = self.memory_index + self.last_batch_buffer_elems\n        memory_insert_indices = tf.range(\n            start=self.memory_index,\n            limit=memory_end_index\n        ) % self.capacity\n\n        for name in sorted(states):\n            assignments.append(tf.scatter_update(\n                ref=self.states_memory[name],\n                indices=memory_insert_indices,\n                # Only buffer elements from batch.\n                updates=states[name][0:self.last_batch_buffer_elems])\n            )\n        for name in sorted(internals):\n            assignments.append(tf.scatter_update(\n                ref=self.internals_buffer[name],\n                indices=memory_insert_indices,\n                updates=internals[name][0:self.last_batch_buffer_elems]\n            ))\n        assignments.append(tf.scatter_update(\n            ref=self.priorities,\n            indices=memory_insert_indices,\n            updates=priorities[0:self.last_batch_buffer_elems]\n        ))\n        assignments.append(tf.scatter_update(\n            ref=self.terminal_memory,\n            indices=memory_insert_indices,\n            updates=terminal[0:self.last_batch_buffer_elems])\n        )\n        assignments.append(tf.scatter_update(\n            ref=self.reward_memory,\n            indices=memory_insert_indices,\n            updates=reward[0:self.last_batch_buffer_elems])\n        )\n        for name in sorted(actions):\n            assignments.append(tf.scatter_update(\n                ref=self.actions_memory[name],\n                indices=memory_insert_indices,\n                updates=actions[name][0:self.last_batch_buffer_elems]\n            ))\n\n        # 4.Update the priorities of the elements already in the memory.\n        # Slice out remaining elements - [] if all batch elements were from buffer.\n        main_memory_priorities = priorities[self.last_batch_buffer_elems:]\n        # Note that priority indices can have a different shape because multiple\n        # samples can be from the same index.\n        main_memory_priorities = main_memory_priorities[0:tf.shape(priority_indices)[0]]\n        assignments.append(tf.scatter_update(\n            ref=self.priorities,\n            indices=priority_indices,\n            updates=main_memory_priorities\n        ))\n\n        with tf.control_dependencies(control_inputs=assignments):\n            # 5. Re-sort memory according to priorities.\n            assignments = list()\n\n            # Obtain sorted order and indices.\n            sorted_priorities, sorted_indices = tf.nn.top_k(\n                input=self.priorities,\n                k=self.capacity,\n                sorted=True\n            )\n            # Re-assign elements according to priorities.\n            # Priorities was the tensor we used to sort, so this can be directly assigned.\n            assignments.append(tf.assign(ref=self.priorities, value=sorted_priorities))\n\n            # All other memory variables are assigned via scatter updates using the indices\n            # returned by the sort:\n            assignments.append(tf.scatter_update(\n                ref=self.terminal_memory,\n                indices=sorted_indices,\n                updates=self.terminal_memory\n            ))\n            for name in sorted(self.states_memory):\n                assignments.append(tf.scatter_update(\n                    ref=self.states_memory[name],\n                    indices=sorted_indices,\n                    updates=self.states_memory[name]\n                ))\n            for name in sorted(self.actions_memory):\n                assignments.append(tf.scatter_update(\n                    ref=self.actions_memory[name],\n                    indices=sorted_indices,\n                    updates=self.actions_memory[name]\n                ))\n            for name in sorted(self.internals_memory):\n                assignments.append(tf.scatter_update(\n                    ref=self.internals_memory[name],\n                    indices=sorted_indices,\n                    updates=self.internals_memory[name]\n                ))\n            assignments.append(tf.scatter_update(\n                ref=self.reward_memory,\n                indices=sorted_indices,\n                updates=self.reward_memory\n            ))\n\n        # 6. Reset buffer index and increment memory index by inserted elements.\n        with tf.control_dependencies(control_inputs=assignments):\n            assignments = list()\n            # Decrement pointer of last elements used.\n            assignments.append(tf.assign_sub(ref=self.buffer_index, value=self.last_batch_buffer_elems))\n\n            # Keep track of memory size as to know whether we can sample from the main memory.\n            # Since the memory pointer can set to 0, we want to know if we are at capacity.\n            total_inserted_elements = self.memory_size + self.last_batch_buffer_elems\n            assignments.append(tf.assign(\n                ref=self.memory_size,\n                value=tf.minimum(x=total_inserted_elements, y=self.capacity))\n            )\n\n            # Update memory insertion index.\n            assignments.append(tf.assign(ref=self.memory_index, value=memory_end_index))\n\n            # Reset batch indices.\n            assignments.append(tf.assign(\n                ref=self.batch_indices,\n                value=tf.zeros(shape=tf.shape(self.batch_indices), dtype=tf.int32)\n            ))\n\n        with tf.control_dependencies(control_inputs=assignments):\n            return tf.no_op()", "response": "Update the priority memory for the current batch."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef setup_components_and_tf_funcs(self, custom_getter=None):\n        # Create network before super-call, since non-empty internals_spec attribute (for RNN) is required subsequently.\n        self.network = Network.from_spec(\n            spec=self.network_spec,\n            kwargs=dict(summary_labels=self.summary_labels)\n        )\n\n        # Now that we have the network component: We can create the internals placeholders.\n        assert len(self.internals_spec) == 0\n        self.internals_spec = self.network.internals_spec()\n        for name in sorted(self.internals_spec):\n            internal = self.internals_spec[name]\n            self.internals_input[name] = tf.placeholder(\n                dtype=util.tf_dtype(internal['type']),\n                shape=(None,) + tuple(internal['shape']),\n                name=('internal-' + name)\n            )\n            if internal['initialization'] == 'zeros':\n                self.internals_init[name] = np.zeros(shape=internal['shape'])\n            else:\n                raise TensorForceError(\"Invalid internal initialization value.\")\n\n        # And only then call super.\n        custom_getter = super(DistributionModel, self).setup_components_and_tf_funcs(custom_getter)\n\n        # Distributions\n        self.distributions = self.create_distributions()\n\n        # KL divergence function\n        self.fn_kl_divergence = tf.make_template(\n            name_='kl-divergence',\n            func_=self.tf_kl_divergence,\n            custom_getter_=custom_getter\n        )\n\n        return custom_getter", "response": "Creates and stores Network and Distribution objects."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates and returns the distributions according to self. distributions_spec.", "response": "def create_distributions(self):\n        \"\"\"\n        Creates and returns the Distribution objects based on self.distributions_spec.\n\n        Returns: Dict of distributions according to self.distributions_spec.\n        \"\"\"\n        distributions = dict()\n        for name in sorted(self.actions_spec):\n            action = self.actions_spec[name]\n\n            if self.distributions_spec is not None and name in self.distributions_spec:\n                kwargs = dict(action)\n                kwargs['scope'] = name\n                kwargs['summary_labels'] = self.summary_labels\n                distributions[name] = Distribution.from_spec(\n                    spec=self.distributions_spec[name],\n                    kwargs=kwargs\n                )\n\n            elif action['type'] == 'bool':\n                distributions[name] = Bernoulli(\n                    shape=action['shape'],\n                    scope=name,\n                    summary_labels=self.summary_labels\n                )\n\n            elif action['type'] == 'int':\n                distributions[name] = Categorical(\n                    shape=action['shape'],\n                    num_actions=action['num_actions'],\n                    scope=name,\n                    summary_labels=self.summary_labels\n                )\n\n            elif action['type'] == 'float':\n                if 'min_value' in action:\n                    distributions[name] = Beta(\n                        shape=action['shape'],\n                        min_value=action['min_value'],\n                        max_value=action['max_value'],\n                        scope=name,\n                        summary_labels=self.summary_labels\n                    )\n\n                else:\n                    distributions[name] = Gaussian(\n                        shape=action['shape'],\n                        scope=name,\n                        summary_labels=self.summary_labels\n                    )\n\n        return distributions"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef from_spec(spec):\n        exploration = util.get_object(\n            obj=spec,\n            predefined_objects=tensorforce.core.explorations.explorations\n        )\n        assert isinstance(exploration, Exploration)\n        return exploration", "response": "Creates an exploration object from a specification dict."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a Memory object from a specification dict.", "response": "def from_spec(spec, kwargs=None):\n        \"\"\"\n        Creates a memory from a specification dict.\n        \"\"\"\n        memory = util.get_object(\n            obj=spec,\n            predefined_objects=tensorforce.core.memories.memories,\n            kwargs=kwargs\n        )\n        assert isinstance(memory, Memory)\n        return memory"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates the TensorFlow operations for performing an optimization step. Args: time: Time tensor. variables: List of variables to optimize. arguments: Dict of arguments for callables, like fn_loss. **kwargs: Additional arguments passed on to the internal optimizer. Returns: List of delta tensors corresponding to the updates for each optimized variable.", "response": "def tf_step(\n        self,\n        time,\n        variables,\n        arguments,\n        **kwargs\n    ):\n        \"\"\"\n        Creates the TensorFlow operations for performing an optimization step.\n\n        Args:\n            time: Time tensor.\n            variables: List of variables to optimize.\n            arguments: Dict of arguments for callables, like fn_loss.\n            **kwargs: Additional arguments passed on to the internal optimizer.\n\n        Returns:\n            List of delta tensors corresponding to the updates for each optimized variable.\n        \"\"\"\n        # Get some (batched) argument to determine batch size.\n        arguments_iter = iter(arguments.values())\n        some_argument = next(arguments_iter)\n\n        try:\n            while not isinstance(some_argument, tf.Tensor) or util.rank(some_argument) == 0:\n                if isinstance(some_argument, dict):\n                    if some_argument:\n                        arguments_iter = iter(some_argument.values())\n                    some_argument = next(arguments_iter)\n                elif isinstance(some_argument, list):\n                    if some_argument:\n                        arguments_iter = iter(some_argument)\n                    some_argument = next(arguments_iter)\n                elif some_argument is None or util.rank(some_argument) == 0:\n                    # Non-batched argument\n                    some_argument = next(arguments_iter)\n                else:\n                    raise TensorForceError(\"Invalid argument type.\")\n        except StopIteration:\n            raise TensorForceError(\"Invalid argument type.\")\n\n        batch_size = tf.shape(input=some_argument)[0]\n        num_samples = tf.cast(\n            x=(self.fraction * tf.cast(x=batch_size, dtype=util.tf_dtype('float'))),\n            dtype=util.tf_dtype('int')\n        )\n        num_samples = tf.maximum(x=num_samples, y=1)\n        indices = tf.random_uniform(shape=(num_samples,), maxval=batch_size, dtype=tf.int32)\n\n        subsampled_arguments = util.map_tensors(\n            fn=(lambda arg: arg if util.rank(arg) == 0 else tf.gather(params=arg, indices=indices)),\n            tensors=arguments\n        )\n\n        return self.optimizer.step(\n            time=time,\n            variables=variables,\n            arguments=subsampled_arguments,\n            **kwargs\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run(self, num_timesteps=None, num_episodes=None, max_episode_timesteps=None, deterministic=False,\n            episode_finished=None, summary_report=None, summary_interval=None, timesteps=None, episodes=None, testing=False, sleep=None\n            ):\n        \"\"\"\n        Args:\n            timesteps (int): Deprecated; see num_timesteps.\n            episodes (int): Deprecated; see num_episodes.\n        \"\"\"\n\n        # deprecation warnings\n        if timesteps is not None:\n            num_timesteps = timesteps\n            warnings.warn(\"WARNING: `timesteps` parameter is deprecated, use `num_timesteps` instead.\",\n                          category=DeprecationWarning)\n        if episodes is not None:\n            num_episodes = episodes\n            warnings.warn(\"WARNING: `episodes` parameter is deprecated, use `num_episodes` instead.\",\n                          category=DeprecationWarning)\n\n        # figure out whether we are using the deprecated way of \"episode_finished\" reporting\n        old_episode_finished = False\n        if episode_finished is not None and len(getargspec(episode_finished).args) == 1:\n            old_episode_finished = True\n\n        # Keep track of episode reward and episode length for statistics.\n        self.start_time = time.time()\n\n        self.agent.reset()\n\n        if num_episodes is not None:\n            num_episodes += self.agent.episode\n\n        if num_timesteps is not None:\n            num_timesteps += self.agent.timestep\n\n        # add progress bar\n        with tqdm(total=num_episodes) as pbar:\n            # episode loop\n            while True:\n                episode_start_time = time.time()\n                state = self.environment.reset()\n                self.agent.reset()\n\n                # Update global counters.\n                self.global_episode = self.agent.episode  # global value (across all agents)\n                self.global_timestep = self.agent.timestep  # global value (across all agents)\n\n                episode_reward = 0\n                self.current_timestep = 0\n\n                # time step (within episode) loop\n                while True:\n                    action = self.agent.act(states=state, deterministic=deterministic)\n\n                    reward = 0\n                    for _ in xrange(self.repeat_actions):\n                        state, terminal, step_reward = self.environment.execute(action=action)\n                        reward += step_reward\n                        if terminal:\n                            break\n\n                    if max_episode_timesteps is not None and self.current_timestep >= max_episode_timesteps:\n                        terminal = True\n\n                    if not testing:\n                        self.agent.observe(terminal=terminal, reward=reward)\n\n                    self.global_timestep += 1\n                    self.current_timestep += 1\n                    episode_reward += reward\n\n                    if terminal or self.agent.should_stop():  # TODO: should_stop also terminate?\n                        break\n\n                    if sleep is not None:\n                        time.sleep(sleep)\n\n                # Update our episode stats.\n                time_passed = time.time() - episode_start_time\n                self.episode_rewards.append(episode_reward)\n                self.episode_timesteps.append(self.current_timestep)\n                self.episode_times.append(time_passed)\n\n                self.global_episode += 1\n                pbar.update(1)\n\n                # Check, whether we should stop this run.\n                if episode_finished is not None:\n                    # deprecated way (passing in only runner object):\n                    if old_episode_finished:\n                        if not episode_finished(self):\n                            break\n                    # new unified way (passing in BaseRunner AND some worker ID):\n                    elif not episode_finished(self, self.id):\n                        break\n                if (num_episodes is not None and self.global_episode >= num_episodes) or \\\n                        (num_timesteps is not None and self.global_timestep >= num_timesteps) or \\\n                        self.agent.should_stop():\n                    break\n            pbar.update(num_episodes - self.global_episode)", "response": "Runs the main loop of the system."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nretrieve the list of experiences for given indices.", "response": "def tf_retrieve_indices(self, indices):\n        \"\"\"\n        Fetches experiences for given indices.\n\n        Args:\n            indices: Index tensor\n\n        Returns: Batch of experiences\n        \"\"\"\n        states = dict()\n        for name in sorted(self.states_memory):\n            states[name] = tf.gather(params=self.states_memory[name], indices=indices)\n\n        internals = dict()\n        for name in sorted(self.internals_memory):\n            internals[name] = tf.gather(params=self.internals_memory[name], indices=indices)\n\n        actions = dict()\n        for name in sorted(self.actions_memory):\n            actions[name] = tf.gather(params=self.actions_memory[name], indices=indices)\n\n        terminal = tf.gather(params=self.terminal_memory, indices=indices)\n        reward = tf.gather(params=self.reward_memory, indices=indices)\n\n        if self.include_next_states:\n            assert util.rank(indices) == 1\n            next_indices = (indices + 1) % self.capacity\n\n            next_states = dict()\n            for name in sorted(self.states_memory):\n                next_states[name] = tf.gather(params=self.states_memory[name], indices=next_indices)\n\n            next_internals = dict()\n            for name in sorted(self.internals_memory):\n                next_internals[name] = tf.gather(params=self.internals_memory[name], indices=next_indices)\n\n            return dict(\n                states=states,\n                internals=internals,\n                actions=actions,\n                terminal=terminal,\n                reward=reward,\n                next_states=next_states,\n                next_internals=next_internals\n            )\n        else:\n            return dict(\n                states=states,\n                internals=internals,\n                actions=actions,\n                terminal=terminal,\n                reward=reward\n            )"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef tf_solve(self, fn_x, x_init, base_value, target_value, estimated_improvement=None):\n        return super(LineSearch, self).tf_solve(fn_x, x_init, base_value, target_value, estimated_improvement)", "response": "Iteratively optimizes the value at x_0 at x_1 and returns the solution to the problem."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninitializing the term for the current iteration.", "response": "def tf_initialize(self, x_init, base_value, target_value, estimated_improvement):\n        \"\"\"\n        Initialization step preparing the arguments for the first iteration of the loop body.\n\n        Args:\n            x_init: Initial solution guess $x_0$.\n            base_value: Value $f(x')$ at $x = x'$.\n            target_value: Value $f(x_0)$ at $x = x_0$.\n            estimated_improvement: Estimated value at $x = x_0$, $f(x')$ if None.\n\n        Returns:\n            Initial arguments for tf_step.\n        \"\"\"\n        self.base_value = base_value\n\n        if estimated_improvement is None:  # TODO: Is this a good alternative?\n            estimated_improvement = tf.abs(x=base_value)\n\n        first_step = super(LineSearch, self).tf_initialize(x_init)\n\n        improvement = tf.divide(\n            x=(target_value - self.base_value),\n            y=tf.maximum(x=estimated_improvement, y=util.epsilon)\n        )\n\n        last_improvement = improvement - 1.0\n\n        if self.mode == 'linear':\n            deltas = [-t * self.parameter for t in x_init]\n            self.estimated_incr = -estimated_improvement * self.parameter\n\n        elif self.mode == 'exponential':\n            deltas = [-t * self.parameter for t in x_init]\n\n        return first_step + (deltas, improvement, last_improvement, estimated_improvement)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef tf_step(self, x, iteration, deltas, improvement, last_improvement, estimated_improvement):\n        x, next_iteration, deltas, improvement, last_improvement, estimated_improvement = super(LineSearch, self).tf_step(\n            x, iteration, deltas, improvement, last_improvement, estimated_improvement\n        )\n\n        next_x = [t + delta for t, delta in zip(x, deltas)]\n\n        if self.mode == 'linear':\n            next_deltas = deltas\n            next_estimated_improvement = estimated_improvement + self.estimated_incr\n\n        elif self.mode == 'exponential':\n            next_deltas = [delta * self.parameter for delta in deltas]\n            next_estimated_improvement = estimated_improvement * self.parameter\n\n        target_value = self.fn_x(next_deltas)\n\n        next_improvement = tf.divide(\n            x=(target_value - self.base_value),\n            y=tf.maximum(x=next_estimated_improvement, y=util.epsilon)\n        )\n\n        return next_x, next_iteration, next_deltas, next_improvement, improvement, next_estimated_improvement", "response": "This method is called by the iteration loop body of the line search algorithm."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef tf_next_step(self, x, iteration, deltas, improvement, last_improvement, estimated_improvement):\n        next_step = super(LineSearch, self).tf_next_step(\n            x, iteration, deltas, improvement, last_improvement, estimated_improvement\n        )\n\n        def undo_deltas():\n            value = self.fn_x([-delta for delta in deltas])\n            with tf.control_dependencies(control_inputs=(value,)):\n                # Trivial operation to enforce control dependency\n                return tf.less(x=value, y=value)  # == False\n\n        improved = tf.cond(\n            pred=(improvement > last_improvement),\n            true_fn=(lambda: True),\n            false_fn=undo_deltas\n        )\n\n        next_step = tf.logical_and(x=next_step, y=improved)\n        next_step = tf.logical_and(x=next_step, y=(improvement < self.accept_ratio))\n        return tf.logical_and(x=next_step, y=(estimated_improvement > util.epsilon))", "response": "Returns a term that checks if another iteration should be performed."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nescape special characters &lt and &gt to HTML - safe sequences.", "response": "def escape(text, quote=False, smart_amp=True):\n    \"\"\"Replace special characters \"&\", \"<\" and \">\" to HTML-safe sequences.\n\n    The original cgi.escape will always escape \"&\", but you can control\n    this one for a smart escape amp.\n\n    :param quote: if set to True, \" and ' will be escaped.\n    :param smart_amp: if set to False, & will always be escaped.\n    \"\"\"\n    if smart_amp:\n        text = _escape_pattern.sub('&amp;', text)\n    else:\n        text = text.replace('&', '&amp;')\n    text = text.replace('<', '&lt;')\n    text = text.replace('>', '&gt;')\n    if quote:\n        text = text.replace('\"', '&quot;')\n        text = text.replace(\"'\", '&#39;')\n    return text"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nremove dangerous URL schemes like javascript: and escape afterwards.", "response": "def escape_link(url):\n    \"\"\"Remove dangerous URL schemes like javascript: and escape afterwards.\"\"\"\n    lower_url = url.lower().strip('\\x00\\x1a \\n\\r\\t')\n    for scheme in _scheme_blacklist:\n        if lower_url.startswith(scheme):\n            return ''\n    return escape(url, quote=True, smart_amp=False)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef markdown(text, escape=True, **kwargs):\n    return Markdown(escape=escape, **kwargs)(text)", "response": "Render markdown formatted text to html."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef hard_wrap(self):\n        self.linebreak = re.compile(r'^ *\\n(?!\\s*$)')\n        self.text = re.compile(\n            r'^[\\s\\S]+?(?=[\\\\<!\\[_*`~]|https?://| *\\n|$)'\n        )", "response": "Grammar for hard wrap linebreak."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nrendering block level code.", "response": "def block_code(self, code, lang=None):\n        \"\"\"Rendering block level code. ``pre > code``.\n\n        :param code: text content of the code block.\n        :param lang: language of the given code.\n        \"\"\"\n        code = code.rstrip('\\n')\n        if not lang:\n            code = escape(code, smart_amp=False)\n            return '<pre><code>%s\\n</code></pre>\\n' % code\n        code = escape(code, quote=True, smart_amp=False)\n        return '<pre><code class=\"lang-%s\">%s\\n</code></pre>\\n' % (lang, code)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrender block level pure html content.", "response": "def block_html(self, html):\n        \"\"\"Rendering block level pure html content.\n\n        :param html: text content of the html snippet.\n        \"\"\"\n        if self.options.get('skip_style') and \\\n           html.lower().startswith('<style'):\n            return ''\n        if self.options.get('escape'):\n            return escape(html)\n        return html"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef list(self, body, ordered=True):\n        tag = 'ul'\n        if ordered:\n            tag = 'ol'\n        return '<%s>\\n%s</%s>\\n' % (tag, body, tag)", "response": "Render list tags like ul and ol."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrender a table cell. Like a table cell. Like a table cell.", "response": "def table_cell(self, content, **flags):\n        \"\"\"Rendering a table cell. Like ``<th>`` ``<td>``.\n\n        :param content: content of current table cell.\n        :param header: whether this is header or not.\n        :param align: align of current table cell.\n        \"\"\"\n        if flags['header']:\n            tag = 'th'\n        else:\n            tag = 'td'\n        align = flags['align']\n        if not align:\n            return '<%s>%s</%s>\\n' % (tag, content, tag)\n        return '<%s style=\"text-align:%s\">%s</%s>\\n' % (\n            tag, align, content, tag\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef codespan(self, text):\n        text = escape(text.rstrip(), smart_amp=False)\n        return '<code>%s</code>' % text", "response": "Rendering inline code text."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef autolink(self, link, is_email=False):\n        text = link = escape(link)\n        if is_email:\n            link = 'mailto:%s' % link\n        return '<a href=\"%s\">%s</a>' % (link, text)", "response": "Render a given link or email address."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef link(self, link, title, text):\n        link = escape_link(link)\n        if not title:\n            return '<a href=\"%s\">%s</a>' % (link, text)\n        title = escape(title, quote=True)\n        return '<a href=\"%s\" title=\"%s\">%s</a>' % (link, title, text)", "response": "Rendering a given link with content and title."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrender a image with title and text.", "response": "def image(self, src, title, text):\n        \"\"\"Rendering a image with title and text.\n\n        :param src: source link of the image.\n        :param title: title text of the image.\n        :param text: alt text of the image.\n        \"\"\"\n        src = escape_link(src)\n        text = escape(text, quote=True)\n        if title:\n            title = escape(title, quote=True)\n            html = '<img src=\"%s\" alt=\"%s\" title=\"%s\"' % (src, text, title)\n        else:\n            html = '<img src=\"%s\" alt=\"%s\"' % (src, text)\n        if self.options.get('use_xhtml'):\n            return '%s />' % html\n        return '%s>' % html"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef footnote_ref(self, key, index):\n        html = (\n            '<sup class=\"footnote-ref\" id=\"fnref-%s\">'\n            '<a href=\"#fn-%s\">%d</a></sup>'\n        ) % (escape(key), escape(key), index)\n        return html", "response": "Renders the ref anchor of a footnote."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrender a footnote item.", "response": "def footnote_item(self, key, text):\n        \"\"\"Rendering a footnote item.\n\n        :param key: identity key for the footnote.\n        :param text: text content of the footnote.\n        \"\"\"\n        back = (\n            '<a href=\"#fnref-%s\" class=\"footnote\">&#8617;</a>'\n        ) % escape(key)\n        text = text.rstrip()\n        if text.endswith('</p>'):\n            text = re.sub(r'<\\/p>$', r'%s</p>' % back, text)\n        else:\n            text = '%s<p>%s</p>' % (text, back)\n        html = '<li id=\"fn-%s\">%s</li>\\n' % (escape(key), text)\n        return html"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef build_metagraph_list(self):\n        ops = []\n\n        self.ignore_unknown_dtypes = True\n        for key in sorted(self.meta_params):\n            value = self.convert_data_to_string(self.meta_params[key])\n\n            if len(value) == 0:\n                continue\n            if isinstance(value, str):\n                ops.append(tf.contrib.summary.generic(name=key, tensor=tf.convert_to_tensor(str(value))))\n            else:\n                ops.append(tf.contrib.summary.generic(name=key, tensor=tf.as_string(tf.convert_to_tensor(value))))\n\n        return ops", "response": "Convert MetaParams into TF Summary Format and create summary_op."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nenabling markdown syntax in docstrings", "response": "def process_docstring(app, what, name, obj, options, lines):\n    \"\"\"Enable markdown syntax in docstrings\"\"\"\n    \n    markdown = \"\\n\".join(lines)\n\n    # ast = cm_parser.parse(markdown)\n    # html = cm_renderer.render(ast)\n    rest = m2r(markdown)\n\n    rest.replace(\"\\r\\n\", \"\\n\")\n    del lines[:]\n    lines.extend(rest.split(\"\\n\"))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate the TensorFlow operations for performing an optimization step. Args: time: Time tensor. variables: List of variables to optimize. arguments: Dict of arguments for callables, like fn_loss. fn_loss: A callable returning the loss of the current model. fn_kl_divergence: A callable returning the KL-divergence relative to the current model. return_estimated_improvement: Returns the estimated improvement resulting from the natural gradient calculation if true. **kwargs: Additional arguments, not used. Returns: List of delta tensors corresponding to the updates for each optimized variable.", "response": "def tf_step(\n        self,\n        time,\n        variables,\n        arguments,\n        fn_loss,\n        fn_kl_divergence,\n        return_estimated_improvement=False,\n        **kwargs\n    ):\n        \"\"\"\n        Creates the TensorFlow operations for performing an optimization step.\n\n        Args:\n            time: Time tensor.\n            variables: List of variables to optimize.\n            arguments: Dict of arguments for callables, like fn_loss.\n            fn_loss: A callable returning the loss of the current model.\n            fn_kl_divergence: A callable returning the KL-divergence relative to the current model.\n            return_estimated_improvement: Returns the estimated improvement resulting from the  \n                natural gradient calculation if true.\n            **kwargs: Additional arguments, not used.\n\n        Returns:\n            List of delta tensors corresponding to the updates for each optimized variable.\n        \"\"\"\n\n        # Optimize: argmin(w) loss(w + delta) such that kldiv(P(w) || P(w + delta)) = learning_rate\n        # For more details, see our blogpost:\n        # https://reinforce.io/blog/end-to-end-computation-graphs-for-reinforcement-learning/\n\n        # from tensorforce import util\n        # arguments = util.map_tensors(fn=tf.stop_gradient, tensors=arguments)\n\n        # kldiv\n        kldiv = fn_kl_divergence(**arguments)\n\n        # grad(kldiv)\n        kldiv_gradients = tf.gradients(ys=kldiv, xs=variables)\n\n        # Calculates the product x * F of a given vector x with the fisher matrix F.\n        # Incorporating the product prevents having to calculate the entire matrix explicitly.\n        def fisher_matrix_product(deltas):\n            # Gradient is not propagated through solver.\n            deltas = [tf.stop_gradient(input=delta) for delta in deltas]\n\n            # delta' * grad(kldiv)\n            delta_kldiv_gradients = tf.add_n(inputs=[\n                tf.reduce_sum(input_tensor=(delta * grad)) for delta, grad in zip(deltas, kldiv_gradients)\n            ])\n\n            # [delta' * F] = grad(delta' * grad(kldiv))\n            return tf.gradients(ys=delta_kldiv_gradients, xs=variables)\n\n        # loss\n        loss = fn_loss(**arguments)\n\n        # grad(loss)\n        loss_gradients = tf.gradients(ys=loss, xs=variables)\n\n        # Solve the following system for delta' via the conjugate gradient solver.\n        # [delta' * F] * delta' = -grad(loss)\n        # --> delta'  (= lambda * delta)\n        deltas = self.solver.solve(fn_x=fisher_matrix_product, x_init=None, b=[-grad for grad in loss_gradients])\n\n        # delta' * F\n        delta_fisher_matrix_product = fisher_matrix_product(deltas=deltas)\n\n        # c' = 0.5 * delta' * F * delta'  (= lambda * c)\n        # TODO: Why constant and hence KL-divergence sometimes negative?\n        constant = 0.5 * tf.add_n(inputs=[\n            tf.reduce_sum(input_tensor=(delta_F * delta))\n            for delta_F, delta in zip(delta_fisher_matrix_product, deltas)\n        ])\n\n        # Natural gradient step if constant > 0\n        def natural_gradient_step():\n            # lambda = sqrt(c' / c)\n            lagrange_multiplier = tf.sqrt(x=(constant / self.learning_rate))\n\n            # delta = delta' / lambda\n            estimated_deltas = [delta / lagrange_multiplier for delta in deltas]\n\n            # improvement = grad(loss) * delta  (= loss_new - loss_old)\n            estimated_improvement = tf.add_n(inputs=[\n                tf.reduce_sum(input_tensor=(grad * delta))\n                for grad, delta in zip(loss_gradients, estimated_deltas)\n            ])\n\n            # Apply natural gradient improvement.\n            applied = self.apply_step(variables=variables, deltas=estimated_deltas)\n\n            with tf.control_dependencies(control_inputs=(applied,)):\n                # Trivial operation to enforce control dependency\n                if return_estimated_improvement:\n                    return [estimated_delta + 0.0 for estimated_delta in estimated_deltas], estimated_improvement\n                else:\n                    return [estimated_delta + 0.0 for estimated_delta in estimated_deltas]\n\n        # Zero step if constant <= 0\n        def zero_step():\n            if return_estimated_improvement:\n                return [tf.zeros_like(tensor=delta) for delta in deltas], 0.0\n            else:\n                return [tf.zeros_like(tensor=delta) for delta in deltas]\n\n        # Natural gradient step only works if constant > 0\n        return tf.cond(pred=(constant > 0.0), true_fn=natural_gradient_step, false_fn=zero_step)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef tf_step(self, time, variables, arguments, fn_reference=None, **kwargs):\n\n        # Set reference to compare with at each optimization step, in case of a comparative loss.\n        arguments['reference'] = fn_reference(**arguments)\n\n        # First step\n        deltas = self.optimizer.step(time=time, variables=variables, arguments=arguments, **kwargs)\n\n        if self.unroll_loop:\n            # Unrolled for loop\n            for _ in xrange(self.num_steps - 1):\n                with tf.control_dependencies(control_inputs=deltas):\n                    step_deltas = self.optimizer.step(time=time, variables=variables, arguments=arguments, **kwargs)\n                    deltas = [delta1 + delta2 for delta1, delta2 in zip(deltas, step_deltas)]\n\n            return deltas\n\n        else:\n            # TensorFlow while loop\n            def body(iteration, deltas):\n                with tf.control_dependencies(control_inputs=deltas):\n                    step_deltas = self.optimizer.step(time=time, variables=variables, arguments=arguments, **kwargs)\n                    deltas = [delta1 + delta2 for delta1, delta2 in zip(deltas, step_deltas)]\n                    return iteration + 1, deltas\n\n            def cond(iteration, deltas):\n                return iteration < self.num_steps - 1\n\n            _, deltas = tf.while_loop(cond=cond, body=body, loop_vars=(0, deltas))\n\n            return deltas", "response": "Creates the TensorFlow operations for performing an optimization step."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the TensorFlow operations for calculating the baseline loss of a batch.", "response": "def tf_baseline_loss(self, states, internals, reward, update, reference=None):\n        \"\"\"\n        Creates the TensorFlow operations for calculating the baseline loss of a batch.\n\n        Args:\n            states: Dict of state tensors.\n            internals: List of prior internal state tensors.\n            reward: Reward tensor.\n            update: Boolean tensor indicating whether this call happens during an update.\n            reference: Optional reference tensor(s), in case of a comparative loss.\n\n        Returns:\n            Loss tensor.\n        \"\"\"\n        if self.baseline_mode == 'states':\n            loss = self.baseline.loss(\n                states=states,\n                internals=internals,\n                reward=reward,\n                update=update,\n                reference=reference\n            )\n\n        elif self.baseline_mode == 'network':\n            loss = self.baseline.loss(\n                states=self.network.apply(x=states, internals=internals, update=update),\n                internals=internals,\n                reward=reward,\n                update=update,\n                reference=reference\n            )\n\n        regularization_loss = self.baseline.regularization_loss()\n        if regularization_loss is not None:\n            loss += regularization_loss\n\n        return loss"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the baseline optimizer arguments as a dict.", "response": "def baseline_optimizer_arguments(self, states, internals, reward):\n        \"\"\"\n        Returns the baseline optimizer arguments including the time, the list of variables to  \n        optimize, and various functions which the optimizer might require to perform an update  \n        step.\n\n        Args:\n            states: Dict of state tensors.\n            internals: List of prior internal state tensors.\n            reward: Reward tensor.\n\n        Returns:\n            Baseline optimizer arguments as dict.\n        \"\"\"\n        arguments = dict(\n            time=self.global_timestep,\n            variables=self.baseline.get_variables(),\n            arguments=dict(\n                states=states,\n                internals=internals,\n                reward=reward,\n                update=tf.constant(value=True),\n            ),\n            fn_reference=self.baseline.reference,\n            fn_loss=self.fn_baseline_loss,\n            # source_variables=self.network.get_variables()\n        )\n        if self.global_model is not None:\n            arguments['global_variables'] = self.global_model.baseline.get_variables()\n        return arguments"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef tf_step(self, time, variables, source_variables, **kwargs):\n        assert all(util.shape(source) == util.shape(target) for source, target in zip(source_variables, variables))\n\n        last_sync = tf.get_variable(\n            name='last-sync',\n            shape=(),\n            dtype=tf.int64,\n            initializer=tf.constant_initializer(value=(-self.sync_frequency), dtype=tf.int64),\n            trainable=False\n        )\n\n        def sync():\n            deltas = list()\n            for source_variable, target_variable in zip(source_variables, variables):\n                delta = self.update_weight * (source_variable - target_variable)\n                deltas.append(delta)\n\n            applied = self.apply_step(variables=variables, deltas=deltas)\n            last_sync_updated = last_sync.assign(value=time)\n\n            with tf.control_dependencies(control_inputs=(applied, last_sync_updated)):\n                # Trivial operation to enforce control dependency\n                return [delta + 0.0 for delta in deltas]\n\n        def no_sync():\n            deltas = list()\n            for variable in variables:\n                delta = tf.zeros(shape=util.shape(variable))\n                deltas.append(delta)\n            return deltas\n\n        do_sync = (time - last_sync >= self.sync_frequency)\n        return tf.cond(pred=do_sync, true_fn=sync, false_fn=no_sync)", "response": "Creates a TensorFlow operation for performing an optimization step."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate the TensorFlow operations for performing an optimization step. Args: time: Time tensor. variables: List of variables to optimize. arguments: Dict of arguments for callables, like fn_loss. fn_loss: A callable returning the loss of the current model. **kwargs: Additional arguments, not used. Returns: List of delta tensors corresponding to the updates for each optimized variable.", "response": "def tf_step(\n        self,\n        time,\n        variables,\n        arguments,\n        fn_loss,\n        **kwargs\n    ):\n        \"\"\"\n        Creates the TensorFlow operations for performing an optimization step.\n\n        Args:\n            time: Time tensor.\n            variables: List of variables to optimize.\n            arguments: Dict of arguments for callables, like fn_loss.\n            fn_loss: A callable returning the loss of the current model.\n            **kwargs: Additional arguments, not used.\n\n        Returns:\n            List of delta tensors corresponding to the updates for each optimized variable.\n        \"\"\"\n        unperturbed_loss = fn_loss(**arguments)\n\n        # First sample\n        perturbations = [tf.random_normal(shape=util.shape(variable)) * self.learning_rate for variable in variables]\n        applied = self.apply_step(variables=variables, deltas=perturbations)\n\n        with tf.control_dependencies(control_inputs=(applied,)):\n            perturbed_loss = fn_loss(**arguments)\n            direction = tf.sign(x=(unperturbed_loss - perturbed_loss))\n            deltas_sum = [direction * perturbation for perturbation in perturbations]\n\n        if self.unroll_loop:\n            # Unrolled for loop\n            previous_perturbations = perturbations\n            for sample in xrange(self.num_samples):\n\n                with tf.control_dependencies(control_inputs=deltas_sum):\n                    perturbations = [tf.random_normal(shape=util.shape(variable)) * self.learning_rate for variable in variables]\n                    perturbation_deltas = [\n                        pert - prev_pert for pert, prev_pert in zip(perturbations, previous_perturbations)\n                    ]\n                    applied = self.apply_step(variables=variables, deltas=perturbation_deltas)\n                    previous_perturbations = perturbations\n\n                with tf.control_dependencies(control_inputs=(applied,)):\n                    perturbed_loss = fn_loss(**arguments)\n                    direction = tf.sign(x=(unperturbed_loss - perturbed_loss))\n                    deltas_sum = [delta + direction * perturbation for delta, perturbation in zip(deltas_sum, perturbations)]\n\n        else:\n            # TensorFlow while loop\n            def body(iteration, deltas_sum, previous_perturbations):\n\n                with tf.control_dependencies(control_inputs=deltas_sum):\n                    perturbations = [tf.random_normal(shape=util.shape(variable)) * self.learning_rate for variable in variables]\n                    perturbation_deltas = [\n                        pert - prev_pert for pert, prev_pert in zip(perturbations, previous_perturbations)\n                    ]\n                    applied = self.apply_step(variables=variables, deltas=perturbation_deltas)\n\n                with tf.control_dependencies(control_inputs=(applied,)):\n                    perturbed_loss = fn_loss(**arguments)\n                    direction = tf.sign(x=(unperturbed_loss - perturbed_loss))\n                    deltas_sum = [delta + direction * perturbation for delta, perturbation in zip(deltas_sum, perturbations)]\n\n                return iteration + 1, deltas_sum, perturbations\n\n            def cond(iteration, deltas_sum, previous_perturbation):\n                return iteration < self.num_samples - 1\n\n            _, deltas_sum, perturbations = tf.while_loop(cond=cond, body=body, loop_vars=(0, deltas_sum, perturbations))\n\n        with tf.control_dependencies(control_inputs=deltas_sum):\n            deltas = [delta / self.num_samples for delta in deltas_sum]\n            perturbation_deltas = [delta - pert for delta, pert in zip(deltas, perturbations)]\n            applied = self.apply_step(variables=variables, deltas=perturbation_deltas)\n\n        with tf.control_dependencies(control_inputs=(applied,)):\n            # Trivial operation to enforce control dependency\n            return [delta + 0.0 for delta in deltas]"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef tf_step(self, time, variables, **kwargs):\n\n        global_variables = kwargs[\"global_variables\"]\n\n        assert all(\n            util.shape(global_variable) == util.shape(local_variable)\n            for global_variable, local_variable in zip(global_variables, variables)\n        )\n\n        local_deltas = self.optimizer.step(time=time, variables=variables, **kwargs)\n\n        with tf.control_dependencies(control_inputs=local_deltas):\n            applied = self.optimizer.apply_step(variables=global_variables, deltas=local_deltas)\n\n        with tf.control_dependencies(control_inputs=(applied,)):\n            update_deltas = list()\n            for global_variable, local_variable in zip(global_variables, variables):\n                delta = global_variable - local_variable\n                update_deltas.append(delta)\n\n            applied = self.apply_step(variables=variables, deltas=update_deltas)\n\n            # TODO: Update time, episode, etc (like in Synchronization)?\n\n        with tf.control_dependencies(control_inputs=(applied,)):\n            return [local_delta + update_delta for local_delta, update_delta in zip(local_deltas, update_deltas)]", "response": "Applies the optimizer step to the set of variables."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef computeStatsEigen(self):\n        # TO-DO: figure out why this op has delays (possibly moving\n        # eigenvectors around?)\n        with tf.device('/cpu:0'):\n            def removeNone(tensor_list):\n                local_list = []\n                for item in tensor_list:\n                    if item is not None:\n                        local_list.append(item)\n                return local_list\n\n            def copyStats(var_list):\n                print(\"copying stats to buffer tensors before eigen decomp\")\n                redundant_stats = {}\n                copied_list = []\n                for item in var_list:\n                    if item is not None:\n                        if item not in redundant_stats:\n                            if self._use_float64:\n                                redundant_stats[item] = tf.cast(\n                                    tf.identity(item), tf.float64)\n                            else:\n                                redundant_stats[item] = tf.identity(item)\n                        copied_list.append(redundant_stats[item])\n                    else:\n                        copied_list.append(None)\n                return copied_list\n            #stats = [copyStats(self.fStats), copyStats(self.bStats)]\n            #stats = [self.fStats, self.bStats]\n\n            stats_eigen = self.stats_eigen\n            computedEigen = {}\n            eigen_reverse_lookup = {}\n            updateOps = []\n            # sync copied stats\n            # with tf.control_dependencies(removeNone(stats[0]) +\n            # removeNone(stats[1])):\n            with tf.control_dependencies([]):\n                for stats_var in stats_eigen:\n                    if stats_var not in computedEigen:\n                        eigens = tf.self_adjoint_eig(stats_var)\n                        e = eigens[0]\n                        Q = eigens[1]\n                        if self._use_float64:\n                            e = tf.cast(e, tf.float32)\n                            Q = tf.cast(Q, tf.float32)\n                        updateOps.append(e)\n                        updateOps.append(Q)\n                        computedEigen[stats_var] = {'e': e, 'Q': Q}\n                        eigen_reverse_lookup[e] = stats_eigen[stats_var]['e']\n                        eigen_reverse_lookup[Q] = stats_eigen[stats_var]['Q']\n\n            self.eigen_reverse_lookup = eigen_reverse_lookup\n            self.eigen_update_list = updateOps\n\n        return updateOps", "response": "compute the eigen decomp using copied var stats"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef tf_step(self, time, variables, **kwargs):\n        fn_loss = kwargs[\"fn_loss\"]\n        if variables is None:\n            variables = tf.trainable_variables\n        return tf.gradients(fn_loss, variables)", "response": "Creates the TensorFlow operations for performing an optimization step on the given variables."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\napplies the given step deltas to the variable values.", "response": "def apply_step(self, variables, deltas, loss_sampled):\n        \"\"\"\n        Applies the given (and already calculated) step deltas to the variable values.\n\n        Args:\n            variables: List of variables.\n            deltas: List of deltas of same length.\n            loss_sampled : the sampled loss\n\n        Returns:\n            The step-applied operation. A tf.group of tf.assign_add ops.\n        \"\"\"\n        update_stats_op = self.compute_and_apply_stats(\n            loss_sampled, var_list=var_list)\n        grads = [(a, b) for a, b in zip(deltas, varlist)]\n        kfacOptim, _ = self.apply_gradients_kfac(grads)\n        return kfacOptim"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nperform an optimization step.", "response": "def minimize(self, time, variables, **kwargs):\n        \"\"\"\n        Performs an optimization step.\n\n        Args:\n            time: Time tensor. Not used for this\n            variables: List of variables to optimize.\n            **kwargs: \n                fn_loss : loss function tensor that is differentiated\n                sampled_loss : the sampled loss from running the model.\n\n        Returns:\n            The optimization operation.\n        \"\"\"\n        loss = kwargs[\"fn_loss\"]\n        sampled_loss = kwargs[\"sampled_loss\"]\n\n        min_op, _ = self.minimize_(loss, sampled_loss, var_list=variables)\n        return min_op"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef setup_components_and_tf_funcs(self, custom_getter=None):\n        custom_getter = super(QDemoModel, self).setup_components_and_tf_funcs(custom_getter)\n\n        self.demo_memory = Replay(\n            states=self.states_spec,\n            internals=self.internals_spec,\n            actions=self.actions_spec,\n            include_next_states=True,\n            capacity=self.demo_memory_capacity,\n            scope='demo-replay',\n            summary_labels=self.summary_labels\n        )\n\n        # Import demonstration optimization.\n        self.fn_import_demo_experience = tf.make_template(\n            name_='import-demo-experience',\n            func_=self.tf_import_demo_experience,\n            custom_getter_=custom_getter\n        )\n\n        # Demonstration loss.\n        self.fn_demo_loss = tf.make_template(\n            name_='demo-loss',\n            func_=self.tf_demo_loss,\n            custom_getter_=custom_getter\n        )\n\n        # Combined loss.\n        self.fn_combined_loss = tf.make_template(\n            name_='combined-loss',\n            func_=self.tf_combined_loss,\n            custom_getter_=custom_getter\n        )\n\n        # Demonstration optimization.\n        self.fn_demo_optimization = tf.make_template(\n            name_='demo-optimization',\n            func_=self.tf_demo_optimization,\n            custom_getter_=custom_getter\n        )\n\n        return custom_getter", "response": "Constructs the extra Replay memory and the demo - loss functions."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nimport a single experience to memory.", "response": "def tf_import_demo_experience(self, states, internals, actions, terminal, reward):\n        \"\"\"\n        Imports a single experience to memory.\n        \"\"\"\n        return self.demo_memory.store(\n            states=states,\n            internals=internals,\n            actions=actions,\n            terminal=terminal,\n            reward=reward\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncreates the demo loss for the states actions terminal and reward.", "response": "def tf_demo_loss(self, states, actions, terminal, reward, internals, update, reference=None):\n        \"\"\"\n        Extends the q-model loss via the dqfd large-margin loss.\n        \"\"\"\n        embedding = self.network.apply(x=states, internals=internals, update=update)\n        deltas = list()\n\n        for name in sorted(actions):\n            action = actions[name]\n            distr_params = self.distributions[name].parameterize(x=embedding)\n            state_action_value = self.distributions[name].state_action_value(distr_params=distr_params, action=action)\n\n            # Create the supervised margin loss\n            # Zero for the action taken, one for all other actions, now multiply by expert margin\n            if self.actions_spec[name]['type'] == 'bool':\n                num_actions = 2\n                action = tf.cast(x=action, dtype=util.tf_dtype('int'))\n            else:\n                num_actions = self.actions_spec[name]['num_actions']\n\n            one_hot = tf.one_hot(indices=action, depth=num_actions)\n            ones = tf.ones_like(tensor=one_hot, dtype=tf.float32)\n            inverted_one_hot = ones - one_hot\n\n            # max_a([Q(s,a) + l(s,a_E,a)], l(s,a_E, a) is 0 for expert action and margin value for others\n            state_action_values = self.distributions[name].state_action_value(distr_params=distr_params)\n            state_action_values = state_action_values + inverted_one_hot * self.expert_margin\n            supervised_selector = tf.reduce_max(input_tensor=state_action_values, axis=-1)\n\n            # J_E(Q) = max_a([Q(s,a) + l(s,a_E,a)] - Q(s,a_E)\n            delta = supervised_selector - state_action_value\n\n            action_size = util.prod(self.actions_spec[name]['shape'])\n            delta = tf.reshape(tensor=delta, shape=(-1, action_size))\n            deltas.append(delta)\n\n        loss_per_instance = tf.reduce_mean(input_tensor=tf.concat(values=deltas, axis=1), axis=1)\n        loss_per_instance = tf.square(x=loss_per_instance)\n\n        return tf.reduce_mean(input_tensor=loss_per_instance, axis=0)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef tf_combined_loss(self, states, internals, actions, terminal, reward, next_states, next_internals, update, reference=None):\n        q_model_loss = self.fn_loss(\n            states=states,\n            internals=internals,\n            actions=actions,\n            terminal=terminal,\n            reward=reward,\n            next_states=next_states,\n            next_internals=next_internals,\n            update=update,\n            reference=reference\n        )\n\n        demo_loss = self.fn_demo_loss(\n            states=states,\n            internals=internals,\n            actions=actions,\n            terminal=terminal,\n            reward=reward,\n            update=update,\n            reference=reference\n        )\n\n        return q_model_loss + self.supervised_weight * demo_loss", "response": "Combines Q - loss and demo loss."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the TensorFlow variables used by the model.", "response": "def get_variables(self, include_submodules=False, include_nontrainable=False):\n        \"\"\"\n        Returns the TensorFlow variables used by the model.\n\n        Returns:\n            List of variables.\n        \"\"\"\n        model_variables = super(QDemoModel, self).get_variables(\n            include_submodules=include_submodules,\n            include_nontrainable=include_nontrainable\n        )\n\n        if include_nontrainable:\n            demo_memory_variables = self.demo_memory.get_variables()\n            model_variables += demo_memory_variables\n\n        return model_variables"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nstoring demonstrations in the demo memory.", "response": "def import_demo_experience(self, states, internals, actions, terminal, reward):\n        \"\"\"\n        Stores demonstrations in the demo memory.\n        \"\"\"\n        fetches = self.import_demo_experience_output\n\n        feed_dict = self.get_feed_dict(\n            states=states,\n            internals=internals,\n            actions=actions,\n            terminal=terminal,\n            reward=reward\n        )\n\n        self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nperforms a demo update by calling the demo optimization operation.", "response": "def demo_update(self):\n        \"\"\"\n        Performs a demonstration update by calling the demo optimization operation.\n        Note that the batch data does not have to be fetched from the demo memory as this is now part of\n        the TensorFlow operation of the demo update.\n        \"\"\"\n        fetches = self.demo_optimization_output\n\n        self.monitored_session.run(fetches=fetches)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef from_config(config, kwargs=None):\n        return util.get_object(\n            obj=config,\n            predefined=tensorforce.core.optimizers.solvers.solvers,\n            kwargs=kwargs\n        )", "response": "Creates a solver from a specification dict."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef tf_step(self, time, variables, **kwargs):\n        arguments = kwargs[\"arguments\"]\n        fn_loss = kwargs[\"fn_loss\"]\n        loss = fn_loss(**arguments)\n\n        # Force loss value to be calculated.\n        with tf.control_dependencies(control_inputs=(loss,)):\n            # Trivial operation to enforce control dependency\n            previous_variables = [variable + 0.0 for variable in variables]\n\n        # The actual tensorflow minimize op.\n        with tf.control_dependencies(control_inputs=previous_variables):\n            applied = self.tf_optimizer.minimize(loss=loss, var_list=variables)  # colocate_gradients_with_ops=True\n\n        # Return deltas after actually having change the variables.\n        with tf.control_dependencies(control_inputs=(applied,)):\n            return [\n                variable - previous_variable\n                for variable, previous_variable in zip(variables, previous_variables)\n            ]", "response": "This function is called by the tensorflow optimizer to perform a single step of the tensorflow optimizer on the current model."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef SetClipboardText(text: str) -> bool:\n    if ctypes.windll.user32.OpenClipboard(0):\n        ctypes.windll.user32.EmptyClipboard()\n        textByteLen = (len(text) + 1) * 2\n        hClipboardData = ctypes.windll.kernel32.GlobalAlloc(0, textByteLen)  # GMEM_FIXED=0\n        hDestText = ctypes.windll.kernel32.GlobalLock(hClipboardData)\n        ctypes.cdll.msvcrt.wcsncpy(ctypes.c_wchar_p(hDestText), ctypes.c_wchar_p(text), textByteLen // 2)\n        ctypes.windll.kernel32.GlobalUnlock(hClipboardData)\n        # system owns hClipboardData after calling SetClipboardData,\n        # application can not write to or free the data once ownership has been transferred to the system\n        ctypes.windll.user32.SetClipboardData(13, hClipboardData)  # CF_TEXT=1, CF_UNICODETEXT=13\n        ctypes.windll.user32.CloseClipboard()\n        return True\n    return False", "response": "Set the text of the clipboard."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nchange the text color on console window.", "response": "def SetConsoleColor(color: int) -> bool:\n    \"\"\"\n    Change the text color on console window.\n    color: int, a value in class `ConsoleColor`.\n    Return bool, True if succeed otherwise False.\n    \"\"\"\n    global _ConsoleOutputHandle\n    global _DefaultConsoleColor\n    if not _DefaultConsoleColor:\n        if not _ConsoleOutputHandle:\n            _ConsoleOutputHandle = ctypes.windll.kernel32.GetStdHandle(_StdOutputHandle)\n        bufferInfo = ConsoleScreenBufferInfo()\n        ctypes.windll.kernel32.GetConsoleScreenBufferInfo(_ConsoleOutputHandle, ctypes.byref(bufferInfo))\n        _DefaultConsoleColor = int(bufferInfo.wAttributes & 0xFF)\n    if sys.stdout:\n        sys.stdout.flush()\n    bool(ctypes.windll.kernel32.SetConsoleTextAttribute(_ConsoleOutputHandle, color))"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nresets the console color to the default color.", "response": "def ResetConsoleColor() -> bool:\n    \"\"\"\n    Reset to the default text color on console window.\n    Return bool, True if succeed otherwise False.\n    \"\"\"\n    if sys.stdout:\n        sys.stdout.flush()\n    bool(ctypes.windll.kernel32.SetConsoleTextAttribute(_ConsoleOutputHandle, _DefaultConsoleColor))"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a native window handle for the given point.", "response": "def WindowFromPoint(x: int, y: int) -> int:\n    \"\"\"\n    WindowFromPoint from Win32.\n    Return int, a native window handle.\n    \"\"\"\n    return ctypes.windll.user32.WindowFromPoint(ctypes.wintypes.POINT(x, y))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetCursorPos() -> tuple:\n    point = ctypes.wintypes.POINT(0, 0)\n    ctypes.windll.user32.GetCursorPos(ctypes.byref(point))\n    return point.x, point.y", "response": "Get current mouse cursor positon."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef SetCursorPos(x: int, y: int) -> bool:\n    return bool(ctypes.windll.user32.SetCursorPos(x, y))", "response": "Set mouse cursor to point x y."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwrapping for Win32. user32. mouse_event.", "response": "def mouse_event(dwFlags: int, dx: int, dy: int, dwData: int, dwExtraInfo: int) -> None:\n    \"\"\"mouse_event from Win32.\"\"\"\n    ctypes.windll.user32.mouse_event(dwFlags, dx, dy, dwData, dwExtraInfo)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef keybd_event(bVk: int, bScan: int, dwFlags: int, dwExtraInfo: int) -> None:\n    ctypes.windll.user32.keybd_event(bVk, bScan, dwFlags, dwExtraInfo)", "response": "Get a new event from the keyboard."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nposts a message to the specified virtual machine.", "response": "def PostMessage(handle: int, msg: int, wParam: int, lParam: int) -> bool:\n    \"\"\"\n    PostMessage from Win32.\n    Return bool, True if succeed otherwise False.\n    \"\"\"\n    return bool(ctypes.windll.user32.PostMessageW(ctypes.c_void_p(handle), msg, wParam, lParam))"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsending a message to the specified target.", "response": "def SendMessage(handle: int, msg: int, wParam: int, lParam: int) -> int:\n    \"\"\"\n    SendMessage from Win32.\n    Return int, the return value specifies the result of the message processing;\n                it depends on the message sent.\n    \"\"\"\n    return ctypes.windll.user32.SendMessageW(ctypes.c_void_p(handle), msg, wParam, lParam)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsimulates mouse click at point x y.", "response": "def Click(x: int, y: int, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    \"\"\"\n    Simulate mouse click at point x, y.\n    x: int.\n    y: int.\n    waitTime: float.\n    \"\"\"\n    SetCursorPos(x, y)\n    screenWidth, screenHeight = GetScreenSize()\n    mouse_event(MouseEventFlag.LeftDown | MouseEventFlag.Absolute, x * 65535 // screenWidth, y * 65535 // screenHeight, 0, 0)\n    time.sleep(0.05)\n    mouse_event(MouseEventFlag.LeftUp | MouseEventFlag.Absolute, x * 65535 // screenWidth, y * 65535 // screenHeight, 0, 0)\n    time.sleep(waitTime)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef MiddleClick(x: int, y: int, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    SetCursorPos(x, y)\n    screenWidth, screenHeight = GetScreenSize()\n    mouse_event(MouseEventFlag.MiddleDown | MouseEventFlag.Absolute, x * 65535 // screenWidth, y * 65535 // screenHeight, 0, 0)\n    time.sleep(0.05)\n    mouse_event(MouseEventFlag.MiddleUp | MouseEventFlag.Absolute, x * 65535 // screenWidth, y * 65535 // screenHeight, 0, 0)\n    time.sleep(waitTime)", "response": "Simulate mouse middle click at point x y."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef RightClick(x: int, y: int, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    SetCursorPos(x, y)\n    screenWidth, screenHeight = GetScreenSize()\n    mouse_event(MouseEventFlag.RightDown | MouseEventFlag.Absolute, x * 65535 // screenWidth, y * 65535 // screenHeight, 0, 0)\n    time.sleep(0.05)\n    mouse_event(MouseEventFlag.RightUp | MouseEventFlag.Absolute, x * 65535 // screenWidth, y * 65535 // screenHeight, 0, 0)\n    time.sleep(waitTime)", "response": "Simulate mouse right click at point x y."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\npresses left mouse. x: int. y: int. waitTime: float.", "response": "def PressMouse(x: int, y: int, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    \"\"\"\n    Press left mouse.\n    x: int.\n    y: int.\n    waitTime: float.\n    \"\"\"\n    SetCursorPos(x, y)\n    screenWidth, screenHeight = GetScreenSize()\n    mouse_event(MouseEventFlag.LeftDown | MouseEventFlag.Absolute, x * 65535 // screenWidth, y * 65535 // screenHeight, 0, 0)\n    time.sleep(waitTime)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ReleaseMouse(waitTime: float = OPERATION_WAIT_TIME) -> None:\n    x, y = GetCursorPos()\n    screenWidth, screenHeight = GetScreenSize()\n    mouse_event(MouseEventFlag.LeftUp | MouseEventFlag.Absolute, x * 65535 // screenWidth, y * 65535 // screenHeight, 0, 0)\n    time.sleep(waitTime)", "response": "Release left mouse.\n    waitTime: float."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef RightPressMouse(x: int, y: int, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    SetCursorPos(x, y)\n    screenWidth, screenHeight = GetScreenSize()\n    mouse_event(MouseEventFlag.RightDown | MouseEventFlag.Absolute, x * 65535 // screenWidth, y * 65535 // screenHeight, 0, 0)\n    time.sleep(waitTime)", "response": "Press right mouse.\n    x: int.\n    y: int.\n    waitTime: float."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef RightReleaseMouse(waitTime: float = OPERATION_WAIT_TIME) -> None:\n    x, y = GetCursorPos()\n    screenWidth, screenHeight = GetScreenSize()\n    mouse_event(MouseEventFlag.RightUp | MouseEventFlag.Absolute, x * 65535 // screenWidth, y * 65535 // screenHeight, 0, 0)\n    time.sleep(waitTime)", "response": "Release right mouse.\n    waitTime: float."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef MoveTo(x: int, y: int, moveSpeed: float = 1, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    if moveSpeed <= 0:\n        moveTime = 0\n    else:\n        moveTime = MAX_MOVE_SECOND / moveSpeed\n    curX, curY = GetCursorPos()\n    xCount = abs(x - curX)\n    yCount = abs(y - curY)\n    maxPoint = max(xCount, yCount)\n    screenWidth, screenHeight = GetScreenSize()\n    maxSide = max(screenWidth, screenHeight)\n    minSide = min(screenWidth, screenHeight)\n    if maxPoint > minSide:\n        maxPoint = minSide\n    if maxPoint < maxSide:\n        maxPoint = 100 + int((maxSide - 100) / maxSide * maxPoint)\n        moveTime = moveTime * maxPoint * 1.0 / maxSide\n    stepCount = maxPoint // 20\n    if stepCount > 1:\n        xStep = (x - curX) * 1.0 / stepCount\n        yStep = (y - curY) * 1.0 / stepCount\n        interval = moveTime / stepCount\n        for i in range(stepCount):\n            cx = curX + int(xStep * i)\n            cy = curY + int(yStep * i)\n            # upper-left(0,0), lower-right(65536,65536)\n            # mouse_event(MouseEventFlag.Move | MouseEventFlag.Absolute, cx*65536//screenWidth, cy*65536//screenHeight, 0, 0)\n            SetCursorPos(cx, cy)\n            time.sleep(interval)\n    SetCursorPos(x, y)\n    time.sleep(waitTime)", "response": "Simulate mouse move to point x y from current cursor."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef DragDrop(x1: int, y1: int, x2: int, y2: int, moveSpeed: float = 1, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    PressMouse(x1, y1, 0.05)\n    MoveTo(x2, y2, moveSpeed, 0.05)\n    ReleaseMouse(waitTime)", "response": "Simulate mouse left button drag from point x1 y1 drop to point x2 y2."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsimulate mouse right button drag drop to point x1 y1 drop to point x2 y2.", "response": "def RightDragDrop(x1: int, y1: int, x2: int, y2: int, moveSpeed: float = 1, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    \"\"\"\n    Simulate mouse right button drag from point x1, y1 drop to point x2, y2.\n    x1: int.\n    y1: int.\n    x2: int.\n    y2: int.\n    moveSpeed: float, 1 normal speed, < 1 move slower, > 1 move faster.\n    waitTime: float.\n    \"\"\"\n    RightPressMouse(x1, y1, 0.05)\n    MoveTo(x2, y2, moveSpeed, 0.05)\n    RightReleaseMouse(waitTime)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef WheelUp(wheelTimes: int = 1, interval: float = 0.05, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    for i in range(wheelTimes):\n        mouse_event(MouseEventFlag.Wheel, 0, 0, 120, 0) #WHEEL_DELTA=120\n        time.sleep(interval)\n    time.sleep(waitTime)", "response": "Simulate mouse wheel up."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GetScreenSize() -> tuple:\n    SM_CXSCREEN = 0\n    SM_CYSCREEN = 1\n    w = ctypes.windll.user32.GetSystemMetrics(SM_CXSCREEN)\n    h = ctypes.windll.user32.GetSystemMetrics(SM_CYSCREEN)\n    return w, h", "response": "Return tuple width height of the screen."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget pixel color of a native window.", "response": "def GetPixelColor(x: int, y: int, handle: int = 0) -> int:\n    \"\"\"\n    Get pixel color of a native window.\n    x: int.\n    y: int.\n    handle: int, the handle of a native window.\n    Return int, the bgr value of point (x,y).\n    r = bgr & 0x0000FF\n    g = (bgr & 0x00FF00) >> 8\n    b = (bgr & 0xFF0000) >> 16\n    If handle is 0, get pixel from Desktop window(root control).\n    Note:\n    Not all devices support GetPixel.\n    An application should call GetDeviceCaps to determine whether a specified device supports this function.\n    For example, console window doesn't support.\n    \"\"\"\n    hdc = ctypes.windll.user32.GetWindowDC(ctypes.c_void_p(handle))\n    bgr = ctypes.windll.gdi32.GetPixel(hdc, x, y)\n    ctypes.windll.user32.ReleaseDC(ctypes.c_void_p(handle), hdc)\n    return bgr"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MessageBox(content: str, title: str, flags: int = MB.Ok) -> int:\n    return ctypes.windll.user32.MessageBoxW(ctypes.c_void_p(0), ctypes.c_wchar_p(content), ctypes.c_wchar_p(title), flags)", "response": "Create a MessageBox from a string."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting the foreground window handle.", "response": "def SetForegroundWindow(handle: int) -> bool:\n    \"\"\"\n    SetForegroundWindow from Win32.\n    handle: int, the handle of a native window.\n    Return bool, True if succeed otherwise False.\n    \"\"\"\n    return bool(ctypes.windll.user32.SetForegroundWindow(ctypes.c_void_p(handle)))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef BringWindowToTop(handle: int) -> bool:\n    return bool(ctypes.windll.user32.BringWindowToTop(ctypes.c_void_p(handle)))", "response": "Bring a window to the top."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nswitching to the current window.", "response": "def SwitchToThisWindow(handle: int) -> None:\n    \"\"\"\n    SwitchToThisWindow from Win32.\n    handle: int, the handle of a native window.\n    \"\"\"\n    ctypes.windll.user32.SwitchToThisWindow(ctypes.c_void_p(handle), 1)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns true if the handle is a top - level window.", "response": "def IsTopLevelWindow(handle: int) -> bool:\n    \"\"\"\n    IsTopLevelWindow from Win32.\n    handle: int, the handle of a native window.\n    Return bool.\n    Only available on Windows 7 or Higher.\n    \"\"\"\n    return bool(ctypes.windll.user32.IsTopLevelWindow(ctypes.c_void_p(handle)))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GetWindowLong(handle: int, index: int) -> int:\n    return ctypes.windll.user32.GetWindowLongW(ctypes.c_void_p(handle), index)", "response": "Get the long value of a window."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nset the value of the specified window.", "response": "def SetWindowLong(handle: int, index: int, value: int) -> int:\n    \"\"\"\n    SetWindowLong from Win32.\n    handle: int, the handle of a native window.\n    index: int.\n    value: int.\n    Return int, the previous value before set.\n    \"\"\"\n    return ctypes.windll.user32.SetWindowLongW(ctypes.c_void_p(handle), index, value)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndetermines if a native window is minimized.", "response": "def IsIconic(handle: int) -> bool:\n    \"\"\"\n    IsIconic from Win32.\n    Determine whether a native window is minimized.\n    handle: int, the handle of a native window.\n    Return bool.\n    \"\"\"\n    return bool(ctypes.windll.user32.IsIconic(ctypes.c_void_p(handle)))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef IsZoomed(handle: int) -> bool:\n    return bool(ctypes.windll.user32.IsZoomed(ctypes.c_void_p(handle)))", "response": "Determines whether a native window is zoomed."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns true if the window is visible.", "response": "def IsWindowVisible(handle: int) -> bool:\n    \"\"\"\n    IsWindowVisible from Win32.\n    handle: int, the handle of a native window.\n    Return bool.\n    \"\"\"\n    return bool(ctypes.windll.user32.IsWindowVisible(ctypes.c_void_p(handle)))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nshow the native window.", "response": "def ShowWindow(handle: int, cmdShow: int) -> bool:\n    \"\"\"\n    ShowWindow from Win32.\n    handle: int, the handle of a native window.\n    cmdShow: int, a value in clas `SW`.\n    Return bool, True if succeed otherwise False.\n    \"\"\"\n    return ctypes.windll.user32.ShowWindow(ctypes.c_void_p(handle), cmdShow)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nmove a window from the current position to the specified size.", "response": "def MoveWindow(handle: int, x: int, y: int, width: int, height: int, repaint: int = 1) -> bool:\n    \"\"\"\n    MoveWindow from Win32.\n    handle: int, the handle of a native window.\n    x: int.\n    y: int.\n    width: int.\n    height: int.\n    repaint: int, use 1 or 0.\n    Return bool, True if succeed otherwise False.\n    \"\"\"\n    return bool(ctypes.windll.user32.MoveWindow(ctypes.c_void_p(handle), x, y, width, height, repaint))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the position of the window in the native window.", "response": "def SetWindowPos(handle: int, hWndInsertAfter: int, x: int, y: int, width: int, height: int, flags: int) -> bool:\n    \"\"\"\n    SetWindowPos from Win32.\n    handle: int, the handle of a native window.\n    hWndInsertAfter: int, a value whose name starts with 'HWND' in class SWP.\n    x: int.\n    y: int.\n    width: int.\n    height: int.\n    flags: int, values whose name starts with 'SWP' in class `SWP`.\n    Return bool, True if succeed otherwise False.\n    \"\"\"\n    return ctypes.windll.user32.SetWindowPos(ctypes.c_void_p(handle), ctypes.c_void_p(hWndInsertAfter), x, y, width, height, flags)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef SetWindowTopmost(handle: int, isTopmost: bool) -> bool:\n    topValue = SWP.HWND_Topmost if isTopmost else SWP.HWND_NoTopmost\n    return bool(SetWindowPos(handle, topValue, 0, 0, 0, 0, SWP.SWP_NoSize | SWP.SWP_NoMove))", "response": "Sets the topmost position of a native window."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting the text of a window.", "response": "def GetWindowText(handle: int) -> str:\n    \"\"\"\n    GetWindowText from Win32.\n    handle: int, the handle of a native window.\n    Return str.\n    \"\"\"\n    arrayType = ctypes.c_wchar * MAX_PATH\n    values = arrayType()\n    ctypes.windll.user32.GetWindowTextW(ctypes.c_void_p(handle), values, MAX_PATH)\n    return values.value"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef SetWindowText(handle: int, text: str) -> bool:\n    return bool(ctypes.windll.user32.SetWindowTextW(ctypes.c_void_p(handle), ctypes.c_wchar_p(text)))", "response": "Sets the text of the window."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets text of a native Win32 Edit.", "response": "def GetEditText(handle: int) -> str:\n    \"\"\"\n    Get text of a native Win32 Edit.\n    handle: int, the handle of a native window.\n    Return str.\n    \"\"\"\n    textLen = SendMessage(handle, 0x000E, 0, 0) + 1  #WM_GETTEXTLENGTH\n    arrayType = ctypes.c_wchar * textLen\n    values = arrayType()\n    SendMessage(handle, 0x000D, textLen, values)  #WM_GETTEXT\n    return values.value"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetConsoleTitle() -> str:\n    arrayType = ctypes.c_wchar * MAX_PATH\n    values = arrayType()\n    ctypes.windll.kernel32.GetConsoleTitleW(values, MAX_PATH)\n    return values.value", "response": "Get the console title from Win32."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the console title from Win32.", "response": "def SetConsoleTitle(text: str) -> bool:\n    \"\"\"\n    SetConsoleTitle from Win32.\n    text: str.\n    Return bool, True if succeed otherwise False.\n    \"\"\"\n    return bool(ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(text)))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef IsDesktopLocked() -> bool:\n    isLocked = False\n    desk = ctypes.windll.user32.OpenDesktopW(ctypes.c_wchar_p('Default'), 0, 0, 0x0100)  # DESKTOP_SWITCHDESKTOP = 0x0100\n    if desk:\n        isLocked = not ctypes.windll.user32.SwitchDesktop(desk)\n        ctypes.windll.user32.CloseDesktop(desk)\n    return isLocked", "response": "Checks if desktop is locked."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nplaying a Wave file.", "response": "def PlayWaveFile(filePath: str = r'C:\\Windows\\Media\\notify.wav', isAsync: bool = False, isLoop: bool = False) -> bool:\n    \"\"\"\n    Call PlaySound from Win32.\n    filePath: str, if emtpy, stop playing the current sound.\n    isAsync: bool, if True, the sound is played asynchronously and returns immediately.\n    isLoop: bool, if True, the sound plays repeatedly until PlayWaveFile(None) is called again, must also set isAsync to True.\n    Return bool, True if succeed otherwise False.\n    \"\"\"\n    if filePath:\n        SND_ASYNC = 0x0001\n        SND_NODEFAULT = 0x0002\n        SND_LOOP = 0x0008\n        SND_FILENAME = 0x20000\n        flags = SND_NODEFAULT | SND_FILENAME\n        if isAsync:\n            flags |= SND_ASYNC\n        if isLoop:\n            flags |= SND_LOOP\n            flags |= SND_ASYNC\n        return bool(ctypes.windll.winmm.PlaySoundW(ctypes.c_wchar_p(filePath), ctypes.c_void_p(0), flags))\n    else:\n        return bool(ctypes.windll.winmm.PlaySoundW(ctypes.c_wchar_p(0), ctypes.c_void_p(0), 0))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef IsProcess64Bit(processId: int) -> bool:\n    try:\n        func = ctypes.windll.ntdll.ZwWow64ReadVirtualMemory64  #only 64 bit OS has this function\n    except Exception as ex:\n        return False\n    try:\n        IsWow64Process = ctypes.windll.kernel32.IsWow64Process\n        IsWow64Process.argtypes = (ctypes.c_void_p, ctypes.POINTER(ctypes.c_int))\n    except Exception as ex:\n        return False\n    hProcess = ctypes.windll.kernel32.OpenProcess(0x1000, 0, processId)  #PROCESS_QUERY_INFORMATION=0x0400,PROCESS_QUERY_LIMITED_INFORMATION=0x1000\n    if hProcess:\n        is64Bit = ctypes.c_int32()\n        if IsWow64Process(hProcess, ctypes.byref(is64Bit)):\n            ctypes.windll.kernel32.CloseHandle(ctypes.c_void_p(hProcess))\n            return False if is64Bit.value else True\n        else:\n            ctypes.windll.kernel32.CloseHandle(ctypes.c_void_p(hProcess))", "response": "Determines if a process is 64 bit."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrun a python script as administrator.", "response": "def RunScriptAsAdmin(argv: list, workingDirectory: str = None, showFlag: int = SW.ShowNormal) -> bool:\n    \"\"\"\n    Run a python script as administrator.\n    System will show a popup dialog askes you whether to elevate as administrator if UAC is enabled.\n    argv: list, a str list like sys.argv, argv[0] is the script file, argv[1:] are other arguments.\n    workingDirectory: str, the working directory for the script file.\n    showFlag: int, a value in class `SW`.\n    Return bool, True if succeed.\n    \"\"\"\n    args = ' '.join('\"{}\"'.format(arg) for arg in argv)\n    return ctypes.windll.shell32.ShellExecuteW(None, \"runas\", sys.executable, args, workingDirectory, showFlag) > 32"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsimulate typing a key.", "response": "def SendKey(key: int, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    \"\"\"\n    Simulate typing a key.\n    key: int, a value in class `Keys`.\n    \"\"\"\n    keybd_event(key, 0, KeyboardEventFlag.KeyDown | KeyboardEventFlag.ExtendedKey, 0)\n    keybd_event(key, 0, KeyboardEventFlag.KeyUp | KeyboardEventFlag.ExtendedKey, 0)\n    time.sleep(waitTime)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsimulates a key down for key.", "response": "def PressKey(key: int, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    \"\"\"\n    Simulate a key down for key.\n    key: int, a value in class `Keys`.\n    waitTime: float.\n    \"\"\"\n    keybd_event(key, 0, KeyboardEventFlag.KeyDown | KeyboardEventFlag.ExtendedKey, 0)\n    time.sleep(waitTime)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ReleaseKey(key: int, waitTime: float = OPERATION_WAIT_TIME) -> None:\n    keybd_event(key, 0, KeyboardEventFlag.KeyUp | KeyboardEventFlag.ExtendedKey, 0)\n    time.sleep(waitTime)", "response": "Simulate a key up for key."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef IsKeyPressed(key: int) -> bool:\n    state = ctypes.windll.user32.GetAsyncKeyState(key)\n    return bool(state & 0x8000)", "response": "Returns True if the key is pressed."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate a new INPUT structure.", "response": "def _CreateInput(structure) -> INPUT:\n    \"\"\"\n    Create Win32 struct `INPUT` for `SendInput`.\n    Return `INPUT`.\n    \"\"\"\n    if isinstance(structure, MOUSEINPUT):\n        return INPUT(InputType.Mouse, _INPUTUnion(mi=structure))\n    if isinstance(structure, KEYBDINPUT):\n        return INPUT(InputType.Keyboard, _INPUTUnion(ki=structure))\n    if isinstance(structure, HARDWAREINPUT):\n        return INPUT(InputType.Hardware, _INPUTUnion(hi=structure))\n    raise TypeError('Cannot create INPUT structure!')"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MouseInput(dx: int, dy: int, mouseData: int = 0, dwFlags: int = MouseEventFlag.LeftDown, time_: int = 0) -> INPUT:\n    return _CreateInput(MOUSEINPUT(dx, dy, mouseData, dwFlags, time_, None))", "response": "Create a new MOUSEINPUT with the given data."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef KeyboardInput(wVk: int, wScan: int, dwFlags: int = KeyboardEventFlag.KeyDown, time_: int = 0) -> INPUT:\n    return _CreateInput(KEYBDINPUT(wVk, wScan, dwFlags, time_, None))", "response": "Create Win32 struct KEYBDINPUT for SendInput."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef HardwareInput(uMsg: int, param: int = 0) -> INPUT:\n    return _CreateInput(HARDWAREINPUT(uMsg, param & 0xFFFF, param >> 16 & 0xFFFF))", "response": "Create Win32 struct HARDWAREINPUT for SendInput."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsending a single unicode character to the keyboard or mouse input stream.", "response": "def SendUnicodeChar(char: str) -> int:\n    \"\"\"\n    Type a single unicode char.\n    char: str, len(char) must equal to 1.\n    Return int, the number of events that it successfully inserted into the keyboard or mouse input stream.\n                If the function returns zero, the input was already blocked by another thread.\n    \"\"\"\n    return SendInput(KeyboardInput(0, ord(char), KeyboardEventFlag.KeyUnicode | KeyboardEventFlag.KeyDown),\n                     KeyboardInput(0, ord(char), KeyboardEventFlag.KeyUnicode | KeyboardEventFlag.KeyUp))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _VKtoSC(key: int) -> int:\n    if key in _SCKeys:\n        return _SCKeys[key]\n    scanCode = ctypes.windll.user32.MapVirtualKeyA(key, 0)\n    if not scanCode:\n        return 0\n    keyList = [Keys.VK_APPS, Keys.VK_CANCEL, Keys.VK_SNAPSHOT, Keys.VK_DIVIDE, Keys.VK_NUMLOCK]\n    if key in keyList:\n        scanCode |= 0x0100\n    return scanCode", "response": "Convert a virtual key to a scan code."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef SendKeys(text: str, interval: float = 0.01, waitTime: float = OPERATION_WAIT_TIME, debug: bool = False) -> None:\n    holdKeys = ('WIN', 'LWIN', 'RWIN', 'SHIFT', 'LSHIFT', 'RSHIFT', 'CTRL', 'CONTROL', 'LCTRL', 'RCTRL', 'LCONTROL', 'LCONTROL', 'ALT', 'LALT', 'RALT')\n    keys = []\n    printKeys = []\n    i = 0\n    insertIndex = 0\n    length = len(text)\n    hold = False\n    include = False\n    lastKeyValue = None\n    while True:\n        if text[i] == '{':\n            rindex = text.find('}', i)\n            if rindex == i + 1:#{}}\n                rindex = text.find('}', i + 2)\n            if rindex == -1:\n                raise ValueError('\"{\" or \"{}\" is not valid, use \"{{}\" for \"{\", use \"{}}\" for \"}\"')\n            key = text[i + 1:rindex]\n            key = [it for it in key.split(' ') if it]\n            if not key:\n                raise ValueError('\"{}\" is not valid, use \"{{Space}}\" or \" \" for \" \"'.format(text[i:rindex + 1]))\n            if (len(key) == 2 and not key[1].isdigit()) or len(key) > 2:\n                raise ValueError('\"{}\" is not valid'.format(text[i:rindex + 1]))\n            upperKey = key[0].upper()\n            count = 1\n            if len(key) > 1:\n                count = int(key[1])\n            for j in range(count):\n                if hold:\n                    if upperKey in SpecialKeyNames:\n                        keyValue = SpecialKeyNames[upperKey]\n                        if type(lastKeyValue) == type(keyValue) and lastKeyValue == keyValue:\n                            insertIndex += 1\n                        printKeys.insert(insertIndex, (key[0], 'KeyDown | ExtendedKey'))\n                        printKeys.insert(insertIndex + 1, (key[0], 'KeyUp | ExtendedKey'))\n                        keys.insert(insertIndex, (keyValue, KeyboardEventFlag.KeyDown | KeyboardEventFlag.ExtendedKey))\n                        keys.insert(insertIndex + 1, (keyValue, KeyboardEventFlag.KeyUp | KeyboardEventFlag.ExtendedKey))\n                        lastKeyValue = keyValue\n                    elif key[0] in CharacterCodes:\n                        keyValue = CharacterCodes[key[0]]\n                        if type(lastKeyValue) == type(keyValue) and lastKeyValue == keyValue:\n                            insertIndex += 1\n                        printKeys.insert(insertIndex, (key[0], 'KeyDown | ExtendedKey'))\n                        printKeys.insert(insertIndex + 1, (key[0], 'KeyUp | ExtendedKey'))\n                        keys.insert(insertIndex, (keyValue, KeyboardEventFlag.KeyDown | KeyboardEventFlag.ExtendedKey))\n                        keys.insert(insertIndex + 1, (keyValue, KeyboardEventFlag.KeyUp | KeyboardEventFlag.ExtendedKey))\n                        lastKeyValue = keyValue\n                    else:\n                        printKeys.insert(insertIndex, (key[0], 'UnicodeChar'))\n                        keys.insert(insertIndex, (key[0], 'UnicodeChar'))\n                        lastKeyValue = key[0]\n                    if include:\n                        insertIndex += 1\n                    else:\n                        if upperKey in holdKeys:\n                            insertIndex += 1\n                        else:\n                            hold = False\n                else:\n                    if upperKey in SpecialKeyNames:\n                        keyValue = SpecialKeyNames[upperKey]\n                        printKeys.append((key[0], 'KeyDown | ExtendedKey'))\n                        printKeys.append((key[0], 'KeyUp | ExtendedKey'))\n                        keys.append((keyValue, KeyboardEventFlag.KeyDown | KeyboardEventFlag.ExtendedKey))\n                        keys.append((keyValue, KeyboardEventFlag.KeyUp | KeyboardEventFlag.ExtendedKey))\n                        lastKeyValue = keyValue\n                        if upperKey in holdKeys:\n                            hold = True\n                            insertIndex = len(keys) - 1\n                        else:\n                            hold = False\n                    else:\n                        printKeys.append((key[0], 'UnicodeChar'))\n                        keys.append((key[0], 'UnicodeChar'))\n                        lastKeyValue = key[0]\n            i = rindex + 1\n        elif text[i] == '(':\n            if hold:\n                include = True\n            else:\n                printKeys.append((text[i], 'UnicodeChar'))\n                keys.append((text[i], 'UnicodeChar'))\n                lastKeyValue = text[i]\n            i += 1\n        elif text[i] == ')':\n            if hold:\n                include = False\n                hold = False\n            else:\n                printKeys.append((text[i], 'UnicodeChar'))\n                keys.append((text[i], 'UnicodeChar'))\n                lastKeyValue = text[i]\n            i += 1\n        else:\n            if hold:\n                if text[i] in CharacterCodes:\n                    keyValue = CharacterCodes[text[i]]\n                    if include and type(lastKeyValue) == type(keyValue) and lastKeyValue == keyValue:\n                        insertIndex += 1\n                    printKeys.insert(insertIndex, (text[i], 'KeyDown | ExtendedKey'))\n                    printKeys.insert(insertIndex + 1, (text[i], 'KeyUp | ExtendedKey'))\n                    keys.insert(insertIndex, (keyValue, KeyboardEventFlag.KeyDown | KeyboardEventFlag.ExtendedKey))\n                    keys.insert(insertIndex + 1, (keyValue, KeyboardEventFlag.KeyUp | KeyboardEventFlag.ExtendedKey))\n                    lastKeyValue = keyValue\n                else:\n                    printKeys.append((text[i], 'UnicodeChar'))\n                    keys.append((text[i], 'UnicodeChar'))\n                    lastKeyValue = text[i]\n                if include:\n                    insertIndex += 1\n                else:\n                    hold = False\n            else:\n                printKeys.append((text[i], 'UnicodeChar'))\n                keys.append((text[i], 'UnicodeChar'))\n                lastKeyValue = text[i]\n            i += 1\n        if i >= length:\n            break\n    hotkeyInterval = 0.01\n    for i, key in enumerate(keys):\n        if key[1] == 'UnicodeChar':\n            SendUnicodeChar(key[0])\n            time.sleep(interval)\n            if debug:\n                Logger.ColorfullyWrite('<Color=DarkGreen>{}</Color>, sleep({})\\n'.format(printKeys[i], interval), writeToFile=False)\n        else:\n            scanCode = _VKtoSC(key[0])\n            keybd_event(key[0], scanCode, key[1], 0)\n            if debug:\n                Logger.Write(printKeys[i], ConsoleColor.DarkGreen, writeToFile=False)\n            if i + 1 == len(keys):\n                time.sleep(interval)\n                if debug:\n                    Logger.Write(', sleep({})\\n'.format(interval), writeToFile=False)\n            else:\n                if key[1] & KeyboardEventFlag.KeyUp:\n                    if keys[i + 1][1] == 'UnicodeChar' or keys[i + 1][1] & KeyboardEventFlag.KeyUp == 0:\n                        time.sleep(interval)\n                        if debug:\n                            Logger.Write(', sleep({})\\n'.format(interval), writeToFile=False)\n                    else:\n                        time.sleep(hotkeyInterval)  #must sleep for a while, otherwise combined keys may not be caught\n                        if debug:\n                            Logger.Write(', sleep({})\\n'.format(hotkeyInterval), writeToFile=False)\n                else:  #KeyboardEventFlag.KeyDown\n                    time.sleep(hotkeyInterval)\n                    if debug:\n                        Logger.Write(', sleep({})\\n'.format(hotkeyInterval), writeToFile=False)\n    #make sure hold keys are not pressed\n    #win = ctypes.windll.user32.GetAsyncKeyState(Keys.VK_LWIN)\n    #ctrl = ctypes.windll.user32.GetAsyncKeyState(Keys.VK_CONTROL)\n    #alt = ctypes.windll.user32.GetAsyncKeyState(Keys.VK_MENU)\n    #shift = ctypes.windll.user32.GetAsyncKeyState(Keys.VK_SHIFT)\n    #if win & 0x8000:\n        #Logger.WriteLine('ERROR: WIN is pressed, it should not be pressed!', ConsoleColor.Red)\n        #keybd_event(Keys.VK_LWIN, 0, KeyboardEventFlag.KeyUp | KeyboardEventFlag.ExtendedKey, 0)\n    #if ctrl & 0x8000:\n        #Logger.WriteLine('ERROR: CTRL is pressed, it should not be pressed!', ConsoleColor.Red)\n        #keybd_event(Keys.VK_CONTROL, 0, KeyboardEventFlag.KeyUp | KeyboardEventFlag.ExtendedKey, 0)\n    #if alt & 0x8000:\n        #Logger.WriteLine('ERROR: ALT is pressed, it should not be pressed!', ConsoleColor.Red)\n        #keybd_event(Keys.VK_MENU, 0, KeyboardEventFlag.KeyUp | KeyboardEventFlag.ExtendedKey, 0)\n    #if shift & 0x8000:\n        #Logger.WriteLine('ERROR: SHIFT is pressed, it should not be pressed!', ConsoleColor.Red)\n        #keybd_event(Keys.VK_SHIFT, 0, KeyboardEventFlag.KeyUp | KeyboardEventFlag.ExtendedKey, 0)\n    time.sleep(waitTime)", "response": "Simulate typing keys on keyboard."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GetPatternIdInterface(patternId: int):\n    global _PatternIdInterfaces\n    if not _PatternIdInterfaces:\n        _PatternIdInterfaces = {\n            # PatternId.AnnotationPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationAnnotationPattern,\n            # PatternId.CustomNavigationPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationCustomNavigationPattern,\n            PatternId.DockPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationDockPattern,\n            # PatternId.DragPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationDragPattern,\n            # PatternId.DropTargetPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationDropTargetPattern,\n            PatternId.ExpandCollapsePattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationExpandCollapsePattern,\n            PatternId.GridItemPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationGridItemPattern,\n            PatternId.GridPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationGridPattern,\n            PatternId.InvokePattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationInvokePattern,\n            PatternId.ItemContainerPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationItemContainerPattern,\n            PatternId.LegacyIAccessiblePattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationLegacyIAccessiblePattern,\n            PatternId.MultipleViewPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationMultipleViewPattern,\n            # PatternId.ObjectModelPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationObjectModelPattern,\n            PatternId.RangeValuePattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationRangeValuePattern,\n            PatternId.ScrollItemPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationScrollItemPattern,\n            PatternId.ScrollPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationScrollPattern,\n            PatternId.SelectionItemPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationSelectionItemPattern,\n            PatternId.SelectionPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationSelectionPattern,\n            # PatternId.SpreadsheetItemPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationSpreadsheetItemPattern,\n            # PatternId.SpreadsheetPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationSpreadsheetPattern,\n            # PatternId.StylesPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationStylesPattern,\n            PatternId.SynchronizedInputPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationSynchronizedInputPattern,\n            PatternId.TableItemPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationTableItemPattern,\n            PatternId.TablePattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationTablePattern,\n            # PatternId.TextChildPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationTextChildPattern,\n            # PatternId.TextEditPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationTextEditPattern,\n            PatternId.TextPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationTextPattern,\n            # PatternId.TextPattern2: _AutomationClient.instance().UIAutomationCore.IUIAutomationTextPattern2,\n            PatternId.TogglePattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationTogglePattern,\n            PatternId.TransformPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationTransformPattern,\n            # PatternId.TransformPattern2: _AutomationClient.instance().UIAutomationCore.IUIAutomationTransformPattern2,\n            PatternId.ValuePattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationValuePattern,\n            PatternId.VirtualizedItemPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationVirtualizedItemPattern,\n            PatternId.WindowPattern: _AutomationClient.instance().UIAutomationCore.IUIAutomationWindowPattern,\n        }\n        debug = False\n        #the following patterns dosn't exist on Windows 7 or lower\n        try:\n            _PatternIdInterfaces[PatternId.AnnotationPattern] = _AutomationClient.instance().UIAutomationCore.IUIAutomationAnnotationPattern\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have AnnotationPattern.', ConsoleColor.Yellow)\n        try:\n            _PatternIdInterfaces[PatternId.CustomNavigationPattern] = _AutomationClient.instance().UIAutomationCore.IUIAutomationCustomNavigationPattern\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have CustomNavigationPattern.', ConsoleColor.Yellow)\n        try:\n            _PatternIdInterfaces[PatternId.DragPattern] = _AutomationClient.instance().UIAutomationCore.IUIAutomationDragPattern\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have DragPattern.', ConsoleColor.Yellow)\n        try:\n            _PatternIdInterfaces[PatternId.DropTargetPattern] = _AutomationClient.instance().UIAutomationCore.IUIAutomationDropTargetPattern\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have DropTargetPattern.', ConsoleColor.Yellow)\n        try:\n            _PatternIdInterfaces[PatternId.ObjectModelPattern] = _AutomationClient.instance().UIAutomationCore.IUIAutomationObjectModelPattern\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have ObjectModelPattern.', ConsoleColor.Yellow)\n        try:\n            _PatternIdInterfaces[PatternId.SpreadsheetItemPattern] = _AutomationClient.instance().UIAutomationCore.IUIAutomationSpreadsheetItemPattern\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have SpreadsheetItemPattern.', ConsoleColor.Yellow)\n        try:\n            _PatternIdInterfaces[PatternId.SpreadsheetPattern] = _AutomationClient.instance().UIAutomationCore.IUIAutomationSpreadsheetPattern\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have SpreadsheetPattern.', ConsoleColor.Yellow)\n        try:\n            _PatternIdInterfaces[PatternId.StylesPattern] = _AutomationClient.instance().UIAutomationCore.IUIAutomationStylesPattern\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have StylesPattern.', ConsoleColor.Yellow)\n        try:\n            _PatternIdInterfaces[PatternId.TextChildPattern] = _AutomationClient.instance().UIAutomationCore.IUIAutomationTextChildPattern\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have TextChildPattern.', ConsoleColor.Yellow)\n        try:\n            _PatternIdInterfaces[PatternId.TextEditPattern] = _AutomationClient.instance().UIAutomationCore.IUIAutomationTextEditPattern\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have TextEditPattern.', ConsoleColor.Yellow)\n        try:\n            _PatternIdInterfaces[PatternId.TextPattern2] = _AutomationClient.instance().UIAutomationCore.IUIAutomationTextPattern2\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have TextPattern2.', ConsoleColor.Yellow)\n        try:\n            _PatternIdInterfaces[PatternId.TransformPattern2] = _AutomationClient.instance().UIAutomationCore.IUIAutomationTransformPattern2\n        except:\n            if debug: Logger.WriteLine('UIAutomationCore does not have TransformPattern2.', ConsoleColor.Yellow)\n    return _PatternIdInterfaces[patternId]", "response": "Gets pattern ID interface by pattern id."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CreatePattern(patternId: int, pattern: ctypes.POINTER(comtypes.IUnknown)):\n    subPattern = pattern.QueryInterface(GetPatternIdInterface(patternId))\n    if subPattern:\n        return PatternConstructors[patternId](pattern=subPattern)", "response": "Create a concreate pattern by pattern id and pattern."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwalks a tree using recursive algorithm.", "response": "def WalkTree(top, getChildren: Callable = None, getFirstChild: Callable = None, getNextSibling: Callable = None, yieldCondition: Callable = None, includeTop: bool = False, maxDepth: int = 0xFFFFFFFF):\n    \"\"\"\n    Walk a tree not using recursive algorithm.\n    top: a tree node.\n    getChildren: function(treeNode) -> list.\n    getNextSibling: function(treeNode) -> treeNode.\n    getNextSibling: function(treeNode) -> treeNode.\n    yieldCondition: function(treeNode, depth) -> bool.\n    includeTop: bool, if True yield top first.\n    maxDepth: int, enum depth.\n\n    If getChildren is valid, ignore getFirstChild and getNextSibling,\n        yield 3 items tuple: (treeNode, depth, remain children count in current depth).\n    If getChildren is not valid, using getFirstChild and getNextSibling,\n        yield 2 items tuple: (treeNode, depth).\n    If yieldCondition is not None, only yield tree nodes that yieldCondition(treeNode, depth)->bool returns True.\n\n    For example:\n    def GetDirChildren(dir_):\n        if os.path.isdir(dir_):\n            return [os.path.join(dir_, it) for it in os.listdir(dir_)]\n    for it, depth, leftCount in WalkTree('D:\\\\', getChildren= GetDirChildren):\n        print(it, depth, leftCount)\n    \"\"\"\n    if maxDepth <= 0:\n        return\n    depth = 0\n    if getChildren:\n        if includeTop:\n            if not yieldCondition or yieldCondition(top, 0):\n                yield top, 0, 0\n        children = getChildren(top)\n        childList = [children]\n        while depth >= 0:   #or while childList:\n            lastItems = childList[-1]\n            if lastItems:\n                if not yieldCondition or yieldCondition(lastItems[0], depth + 1):\n                    yield lastItems[0], depth + 1, len(lastItems) - 1\n                if depth + 1 < maxDepth:\n                    children = getChildren(lastItems[0])\n                    if children:\n                        depth += 1\n                        childList.append(children)\n                del lastItems[0]\n            else:\n                del childList[depth]\n                depth -= 1\n    elif getFirstChild and getNextSibling:\n        if includeTop:\n            if not yieldCondition or yieldCondition(top, 0):\n                yield top, 0\n        child = getFirstChild(top)\n        childList = [child]\n        while depth >= 0:  #or while childList:\n            lastItem = childList[-1]\n            if lastItem:\n                if not yieldCondition or yieldCondition(lastItem, depth + 1):\n                    yield lastItem, depth + 1\n                child = getNextSibling(lastItem)\n                childList[depth] = child\n                if depth + 1 < maxDepth:\n                    child = getFirstChild(lastItem)\n                    if child:\n                        depth += 1\n                        childList.append(child)\n            else:\n                del childList[depth]\n                depth -= 1"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a control instance from a point.", "response": "def ControlFromPoint(x: int, y: int) -> Control:\n    \"\"\"\n    Call IUIAutomation ElementFromPoint x,y. May return None if mouse is over cmd's title bar icon.\n    Return `Control` subclass or None.\n    \"\"\"\n    element = _AutomationClient.instance().IUIAutomation.ElementFromPoint(ctypes.wintypes.POINT(x, y))\n    return Control.CreateControlFromElement(element)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a control from a native handle from point x y and call IUIAutomation. ElementFromHandle.", "response": "def ControlFromPoint2(x: int, y: int) -> Control:\n    \"\"\"\n    Get a native handle from point x,y and call IUIAutomation.ElementFromHandle.\n    Return `Control` subclass.\n    \"\"\"\n    return Control.CreateControlFromElement(_AutomationClient.instance().IUIAutomation.ElementFromHandle(WindowFromPoint(x, y)))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ControlFromHandle(handle: int) -> Control:\n    return Control.CreateControlFromElement(_AutomationClient.instance().IUIAutomation.ElementFromHandle(handle))", "response": "Return a control from a native handle."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning True if control1 and control2 represent the same control otherwise False.", "response": "def ControlsAreSame(control1: Control, control2: Control) -> bool:\n    \"\"\"\n    control1: `Control` or its subclass.\n    control2: `Control` or its subclass.\n    Return bool, True if control1 and control2 represent the same control otherwise False.\n    \"\"\"\n    return bool(_AutomationClient.instance().IUIAutomation.CompareElements(control1.Element, control2.Element))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef WalkControl(control: Control, includeTop: bool = False, maxDepth: int = 0xFFFFFFFF):\n    if includeTop:\n        yield control, 0\n    if maxDepth <= 0:\n        return\n    depth = 0\n    child = control.GetFirstChildControl()\n    controlList = [child]\n    while depth >= 0:\n        lastControl = controlList[-1]\n        if lastControl:\n            yield lastControl, depth + 1\n            child = lastControl.GetNextSiblingControl()\n            controlList[depth] = child\n            if depth + 1 < maxDepth:\n                child = lastControl.GetFirstChildControl()\n                if child:\n                    depth += 1\n                    controlList.append(child)\n        else:\n            del controlList[depth]\n            depth -= 1", "response": "Walks a control and returns an iterator over the control s children."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nprints and log a single control.", "response": "def LogControl(control: Control, depth: int = 0, showAllName: bool = True) -> None:\n    \"\"\"\n    Print and log control's properties.\n    control: `Control` or its subclass.\n    depth: int, current depth.\n    showAllName: bool, if False, print the first 30 characters of control.Name.\n    \"\"\"\n    def getKeyName(theDict, theValue):\n        for key in theDict:\n            if theValue == theDict[key]:\n                return key\n    indent = ' ' * depth * 4\n    Logger.Write('{0}ControlType: '.format(indent))\n    Logger.Write(control.ControlTypeName, ConsoleColor.DarkGreen)\n    Logger.Write('    ClassName: ')\n    Logger.Write(control.ClassName, ConsoleColor.DarkGreen)\n    Logger.Write('    AutomationId: ')\n    Logger.Write(control.AutomationId, ConsoleColor.DarkGreen)\n    Logger.Write('    Rect: ')\n    Logger.Write(control.BoundingRectangle, ConsoleColor.DarkGreen)\n    Logger.Write('    Name: ')\n    Logger.Write(control.Name, ConsoleColor.DarkGreen, printTruncateLen=0 if showAllName else 30)\n    Logger.Write('    Handle: ')\n    Logger.Write('0x{0:X}({0})'.format(control.NativeWindowHandle), ConsoleColor.DarkGreen)\n    Logger.Write('    Depth: ')\n    Logger.Write(depth, ConsoleColor.DarkGreen)\n    supportedPatterns = list(filter(lambda t: t[0], ((control.GetPattern(id_), name) for id_, name in PatternIdNames.items())))\n    for pt, name in supportedPatterns:\n        if isinstance(pt, ValuePattern):\n            Logger.Write('    ValuePattern.Value: ')\n            Logger.Write(pt.Value, ConsoleColor.DarkGreen, printTruncateLen=0 if showAllName else 30)\n        elif isinstance(pt, RangeValuePattern):\n            Logger.Write('    RangeValuePattern.Value: ')\n            Logger.Write(pt.Value, ConsoleColor.DarkGreen)\n        elif isinstance(pt, TogglePattern):\n            Logger.Write('    TogglePattern.ToggleState: ')\n            Logger.Write('ToggleState.' + getKeyName(ToggleState.__dict__, pt.ToggleState), ConsoleColor.DarkGreen)\n        elif isinstance(pt, SelectionItemPattern):\n            Logger.Write('    SelectionItemPattern.IsSelected: ')\n            Logger.Write(pt.IsSelected, ConsoleColor.DarkGreen)\n        elif isinstance(pt, ExpandCollapsePattern):\n            Logger.Write('    ExpandCollapsePattern.ExpandCollapseState: ')\n            Logger.Write('ExpandCollapseState.' + getKeyName(ExpandCollapseState.__dict__, pt.ExpandCollapseState), ConsoleColor.DarkGreen)\n        elif isinstance(pt, ScrollPattern):\n            Logger.Write('    ScrollPattern.HorizontalScrollPercent: ')\n            Logger.Write(pt.HorizontalScrollPercent, ConsoleColor.DarkGreen)\n            Logger.Write('    ScrollPattern.VerticalScrollPercent: ')\n            Logger.Write(pt.VerticalScrollPercent, ConsoleColor.DarkGreen)\n        elif isinstance(pt, GridPattern):\n            Logger.Write('    GridPattern.RowCount: ')\n            Logger.Write(pt.RowCount, ConsoleColor.DarkGreen)\n            Logger.Write('    GridPattern.ColumnCount: ')\n            Logger.Write(pt.ColumnCount, ConsoleColor.DarkGreen)\n        elif isinstance(pt, GridItemPattern):\n            Logger.Write('    GridItemPattern.Row: ')\n            Logger.Write(pt.Column, ConsoleColor.DarkGreen)\n            Logger.Write('    GridItemPattern.Column: ')\n            Logger.Write(pt.Column, ConsoleColor.DarkGreen)\n        elif isinstance(pt, TextPattern):\n            Logger.Write('    TextPattern.Text: ')\n            Logger.Write(pt.DocumentRange.GetText(30), ConsoleColor.DarkGreen)\n    Logger.Write('    SupportedPattern:')\n    for pt, name in supportedPatterns:\n        Logger.Write(' ' + name, ConsoleColor.DarkGreen)\n    Logger.Write('\\n')"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef EnumAndLogControl(control: Control, maxDepth: int = 0xFFFFFFFF, showAllName: bool = True, startDepth: int = 0) -> None:\n    for c, d in WalkControl(control, True, maxDepth):\n        LogControl(c, d + startDepth, showAllName)", "response": "Enumerates and logs a control and its descendants."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef EnumAndLogControlAncestors(control: Control, showAllName: bool = True) -> None:\n    lists = []\n    while control:\n        lists.insert(0, control)\n        control = control.GetParentControl()\n    for i, control in enumerate(lists):\n        LogControl(control, i, showAllName)", "response": "Enumerates and logs the ancestors of a control."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef FindControl(control: Control, compare: Callable, maxDepth: int = 0xFFFFFFFF, findFromSelf: bool = False, foundIndex: int = 1) -> Control:\n    foundCount = 0\n    if not control:\n        control = GetRootControl()\n    traverseCount = 0\n    for child, depth in WalkControl(control, findFromSelf, maxDepth):\n        traverseCount += 1\n        if compare(child, depth):\n            foundCount += 1\n            if foundCount == foundIndex:\n                child.traverseCount = traverseCount\n                return child", "response": "Walks a control and returns a new control or None if not found."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwaits for a given hotkey to be released.", "response": "def WaitHotKeyReleased(hotkey: tuple) -> None:\n    \"\"\"hotkey: tuple, two ints tuple(modifierKey, key)\"\"\"\n    mod = {ModifierKey.Alt: Keys.VK_MENU,\n           ModifierKey.Control: Keys.VK_CONTROL,\n                 ModifierKey.Shift: Keys.VK_SHIFT,\n                 ModifierKey.Win: Keys.VK_LWIN\n           }\n    while True:\n        time.sleep(0.05)\n        if IsKeyPressed(hotkey[1]):\n            continue\n        for k, v in mod.items():\n            if k & hotkey[0]:\n                if IsKeyPressed(v):\n                    break\n        else:\n            break"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef RunByHotKey(keyFunctions: dict, stopHotKey: tuple = None, exitHotKey: tuple = (ModifierKey.Control, Keys.VK_D), waitHotKeyReleased: bool = True) -> None:\n    from threading import Thread, Event, currentThread\n    import traceback\n\n    def getModName(theDict, theValue):\n        name = ''\n        for key in theDict:\n            if isinstance(theDict[key], int) and theValue & theDict[key]:\n                if name:\n                    name += '|'\n                name += key\n        return name\n    def getKeyName(theDict, theValue):\n        for key in theDict:\n            if theValue == theDict[key]:\n                return key\n    def releaseAllKey():\n        for key, value in Keys.__dict__.items():\n            if isinstance(value, int) and key.startswith('VK'):\n                if IsKeyPressed(value):\n                    ReleaseKey(value)\n    def threadFunc(function, stopEvent, hotkey, hotkeyName):\n        if waitHotKeyReleased:\n            WaitHotKeyReleased(hotkey)\n        try:\n            function(stopEvent)\n        except Exception as ex:\n            Logger.ColorfullyWrite('Catch an exception <Color=Red>{}</Color> in thread for hotkey <Color=DarkCyan>{}</Color>\\n'.format(\n                ex.__class__.__name__, hotkeyName), writeToFile=False)\n            print(traceback.format_exc())\n        finally:\n            releaseAllKey()  #need to release keys if some keys were pressed\n            Logger.ColorfullyWrite('{} for function <Color=DarkCyan>{}</Color> exits, hotkey <Color=DarkCyan>{}</Color>\\n'.format(\n                currentThread(), function.__name__, hotkeyName), ConsoleColor.DarkYellow, writeToFile=False)\n\n    stopHotKeyId = 1\n    exitHotKeyId = 2\n    hotKeyId = 3\n    registed = True\n    id2HotKey = {}\n    id2Function = {}\n    id2Thread = {}\n    id2Name = {}\n    for hotkey in keyFunctions:\n        id2HotKey[hotKeyId] = hotkey\n        id2Function[hotKeyId] = keyFunctions[hotkey]\n        id2Thread[hotKeyId] = None\n        modName = getModName(ModifierKey.__dict__, hotkey[0])\n        keyName = getKeyName(Keys.__dict__, hotkey[1])\n        id2Name[hotKeyId] = str((modName, keyName))\n        if ctypes.windll.user32.RegisterHotKey(0, hotKeyId, hotkey[0], hotkey[1]):\n            Logger.ColorfullyWrite('Register hotkey <Color=DarkGreen>{}</Color> successfully\\n'.format((modName, keyName)), writeToFile=False)\n        else:\n            registed = False\n            Logger.ColorfullyWrite('Register hotkey <Color=DarkGreen>{}</Color> unsuccessfully, maybe it was allready registered by another program\\n'.format((modName, keyName)), writeToFile=False)\n        hotKeyId += 1\n    if stopHotKey and len(stopHotKey) == 2:\n        modName = getModName(ModifierKey.__dict__, stopHotKey[0])\n        keyName = getKeyName(Keys.__dict__, stopHotKey[1])\n        if ctypes.windll.user32.RegisterHotKey(0, stopHotKeyId, stopHotKey[0], stopHotKey[1]):\n            Logger.ColorfullyWrite('Register stop hotkey <Color=DarkYellow>{}</Color> successfully\\n'.format((modName, keyName)), writeToFile=False)\n        else:\n            registed = False\n            Logger.ColorfullyWrite('Register stop hotkey <Color=DarkYellow>{}</Color> unsuccessfully, maybe it was allready registered by another program\\n'.format((modName, keyName)), writeToFile=False)\n    if not registed:\n        return\n    if exitHotKey and len(exitHotKey) == 2:\n        modName = getModName(ModifierKey.__dict__, exitHotKey[0])\n        keyName = getKeyName(Keys.__dict__, exitHotKey[1])\n        if ctypes.windll.user32.RegisterHotKey(0, exitHotKeyId, exitHotKey[0], exitHotKey[1]):\n            Logger.ColorfullyWrite('Register exit hotkey <Color=DarkYellow>{}</Color> successfully\\n'.format((modName, keyName)), writeToFile=False)\n        else:\n            Logger.ColorfullyWrite('Register exit hotkey <Color=DarkYellow>{}</Color> unsuccessfully\\n'.format((modName, keyName)), writeToFile=False)\n    funcThread = None\n    livingThreads = []\n    stopEvent = Event()\n    msg = ctypes.wintypes.MSG()\n    while ctypes.windll.user32.GetMessageW(ctypes.byref(msg), ctypes.c_void_p(0), 0, 0) != 0:\n        if msg.message == 0x0312: # WM_HOTKEY=0x0312\n            if msg.wParam in id2HotKey:\n                if msg.lParam & 0x0000FFFF == id2HotKey[msg.wParam][0] and msg.lParam >> 16 & 0x0000FFFF == id2HotKey[msg.wParam][1]:\n                    Logger.ColorfullyWrite('----------hotkey <Color=DarkGreen>{}</Color> pressed----------\\n'.format(id2Name[msg.wParam]), writeToFile=False)\n                    if not id2Thread[msg.wParam]:\n                        stopEvent.clear()\n                        funcThread = Thread(None, threadFunc, args=(id2Function[msg.wParam], stopEvent, id2HotKey[msg.wParam], id2Name[msg.wParam]))\n                        funcThread.start()\n                        id2Thread[msg.wParam] = funcThread\n                    else:\n                        if id2Thread[msg.wParam].is_alive():\n                            Logger.WriteLine('There is a {} that is already running for hotkey {}'.format(id2Thread[msg.wParam], id2Name[msg.wParam]), ConsoleColor.Yellow, writeToFile=False)\n                        else:\n                            stopEvent.clear()\n                            funcThread = Thread(None, threadFunc, args=(id2Function[msg.wParam], stopEvent, id2HotKey[msg.wParam], id2Name[msg.wParam]))\n                            funcThread.start()\n                            id2Thread[msg.wParam] = funcThread\n            elif stopHotKeyId == msg.wParam:\n                if msg.lParam & 0x0000FFFF == stopHotKey[0] and msg.lParam >> 16 & 0x0000FFFF == stopHotKey[1]:\n                    Logger.Write('----------stop hotkey pressed----------\\n', ConsoleColor.DarkYellow, writeToFile=False)\n                    stopEvent.set()\n                    for id_ in id2Thread:\n                        if id2Thread[id_]:\n                            if id2Thread[id_].is_alive():\n                                livingThreads.append((id2Thread[id_], id2Name[id_]))\n                            id2Thread[id_] = None\n            elif exitHotKeyId == msg.wParam:\n                if msg.lParam & 0x0000FFFF == exitHotKey[0] and msg.lParam >> 16 & 0x0000FFFF == exitHotKey[1]:\n                    Logger.Write('Exit hotkey pressed. Exit\\n', ConsoleColor.DarkYellow, writeToFile=False)\n                    stopEvent.set()\n                    for id_ in id2Thread:\n                        if id2Thread[id_]:\n                            if id2Thread[id_].is_alive():\n                                livingThreads.append((id2Thread[id_], id2Name[id_]))\n                            id2Thread[id_] = None\n                    break\n    for thread, hotkeyName in livingThreads:\n        if thread.is_alive():\n            Logger.Write('join {} triggered by hotkey {}\\n'.format(thread, hotkeyName), ConsoleColor.DarkYellow, writeToFile=False)\n            thread.join(2)\n    os._exit(0)", "response": "Runs a function on the main thread of the uiautomator."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwrites a log to stdout.", "response": "def Write(log: Any, consoleColor: int = ConsoleColor.Default, writeToFile: bool = True, printToStdout: bool = True, logFile: str = None, printTruncateLen: int = 0) -> None:\n        \"\"\"\n        log: any type.\n        consoleColor: int, a value in class `ConsoleColor`, such as `ConsoleColor.DarkGreen`.\n        writeToFile: bool.\n        printToStdout: bool.\n        logFile: str, log file path.\n        printTruncateLen: int, if <= 0, log is not truncated when print.\n        \"\"\"\n        if not isinstance(log, str):\n            log = str(log)\n        if printToStdout and sys.stdout:\n            isValidColor = (consoleColor >= ConsoleColor.Black and consoleColor <= ConsoleColor.White)\n            if isValidColor:\n                SetConsoleColor(consoleColor)\n            try:\n                if printTruncateLen > 0 and len(log) > printTruncateLen:\n                    sys.stdout.write(log[:printTruncateLen] + '...')\n                else:\n                    sys.stdout.write(log)\n            except Exception as ex:\n                SetConsoleColor(ConsoleColor.Red)\n                isValidColor = True\n                sys.stdout.write(ex.__class__.__name__ + ': can\\'t print the log!')\n                if log.endswith('\\n'):\n                    sys.stdout.write('\\n')\n            if isValidColor:\n                ResetConsoleColor()\n            sys.stdout.flush()\n        if not writeToFile:\n            return\n        fileName = logFile if logFile else Logger.FileName\n        try:\n            fout = open(fileName, 'a+', encoding='utf-8')\n            fout.write(log)\n        except Exception as ex:\n            if sys.stdout:\n                sys.stdout.write(ex.__class__.__name__ + ': can\\'t write the log!')\n        finally:\n            if fout:\n                fout.close()"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef WriteLine(log: Any, consoleColor: int = -1, writeToFile: bool = True, printToStdout: bool = True, logFile: str = None) -> None:\n        Logger.Write('{}\\n'.format(log), consoleColor, writeToFile, printToStdout, logFile)", "response": "Writes a log line to stdout or to a log file."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ColorfullyWriteLine(log: str, consoleColor: int = -1, writeToFile: bool = True, printToStdout: bool = True, logFile: str = None) -> None:\n        Logger.ColorfullyWrite(log + '\\n', consoleColor, writeToFile, printToStdout, logFile)", "response": "Colorfully writes a log line."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Log(log: Any = '', consoleColor: int = -1, writeToFile: bool = True, printToStdout: bool = True, logFile: str = None) -> None:\n        t = datetime.datetime.now()\n        frame = sys._getframe(1)\n        log = '{}-{:02}-{:02} {:02}:{:02}:{:02}.{:03} Function: {}, Line: {} -> {}\\n'.format(t.year, t.month, t.day,\n            t.hour, t.minute, t.second, t.microsecond // 1000, frame.f_code.co_name, frame.f_lineno, log)\n        Logger.Write(log, consoleColor, writeToFile, printToStdout, logFile)", "response": "Writes a log to stdout."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef FromHandle(self, hwnd: int, left: int = 0, top: int = 0, right: int = 0, bottom: int = 0) -> bool:\n        self.Release()\n        root = GetRootControl()\n        rect = ctypes.wintypes.RECT()\n        ctypes.windll.user32.GetWindowRect(hwnd, ctypes.byref(rect))\n        left, top, right, bottom = left + rect.left, top + rect.top, right + rect.left, bottom + rect.top\n        self._bitmap = _DllClient.instance().dll.BitmapFromWindow(root.NativeWindowHandle, left, top, right, bottom)\n        self._getsize()\n        return self._bitmap > 0", "response": "Capture a native window to Bitmap by its handle."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a Bitmap from a control.", "response": "def FromControl(self, control: 'Control', x: int = 0, y: int = 0, width: int = 0, height: int = 0) -> bool:\n        \"\"\"\n        Capture a control to Bitmap.\n        control: `Control` or its subclass.\n        x: int.\n        y: int.\n        width: int.\n        height: int.\n        x, y: the point in control's internal position(from 0,0)\n        width, height: image's width and height from x, y, use 0 for entire area,\n        If width(or height) < 0, image size will be control's width(or height) - width(or height).\n        Return bool, True if succeed otherwise False.\n        \"\"\"\n        rect = control.BoundingRectangle\n        while rect.width() == 0 or rect.height() == 0:\n            #some controls maybe visible but their BoundingRectangle are all 0, capture its parent util valid\n            control = control.GetParentControl()\n            if not control:\n                return False\n            rect = control.BoundingRectangle\n        if width <= 0:\n            width = rect.width() + width\n        if height <= 0:\n            height = rect.height() + height\n        handle = control.NativeWindowHandle\n        if handle:\n            left = x\n            top = y\n            right = left + width\n            bottom = top + height\n        else:\n            while True:\n                control = control.GetParentControl()\n                handle = control.NativeWindowHandle\n                if handle:\n                    pRect = control.BoundingRectangle\n                    left = rect.left - pRect.left + x\n                    top = rect.top - pRect.top + y\n                    right = left + width\n                    bottom = top + height\n                    break\n        return self.FromHandle(handle, left, top, right, bottom)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nloads an image from a file.", "response": "def FromFile(self, filePath: str) -> bool:\n        \"\"\"\n        Load image from a file.\n        filePath: str.\n        Return bool, True if succeed otherwise False.\n        \"\"\"\n        self.Release()\n        self._bitmap = _DllClient.instance().dll.BitmapFromFile(ctypes.c_wchar_p(filePath))\n        self._getsize()\n        return self._bitmap > 0"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsaving to a file.", "response": "def ToFile(self, savePath: str) -> bool:\n        \"\"\"\n        Save to a file.\n        savePath: str, should end with .bmp, .jpg, .jpeg, .png, .gif, .tif, .tiff.\n        Return bool, True if succeed otherwise False.\n        \"\"\"\n        name, ext = os.path.splitext(savePath)\n        extMap = {'.bmp': 'image/bmp'\n                  , '.jpg': 'image/jpeg'\n                  , '.jpeg': 'image/jpeg'\n                  , '.gif': 'image/gif'\n                  , '.tif': 'image/tiff'\n                  , '.tiff': 'image/tiff'\n                  , '.png': 'image/png'\n                  }\n        gdiplusImageFormat = extMap.get(ext.lower(), 'image/png')\n        return bool(_DllClient.instance().dll.BitmapToFile(self._bitmap, ctypes.c_wchar_p(savePath), ctypes.c_wchar_p(gdiplusImageFormat)))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetPixelColor(self, x: int, y: int) -> int:\n        return _DllClient.instance().dll.BitmapGetPixel(self._bitmap, x, y)", "response": "Get color value of a pixel."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting the color value of a pixel.", "response": "def SetPixelColor(self, x: int, y: int, argb: int) -> bool:\n        \"\"\"\n        Set color value of a pixel.\n        x: int.\n        y: int.\n        argb: int, color value.\n        Return bool, True if succeed otherwise False.\n        \"\"\"\n        return _DllClient.instance().dll.BitmapSetPixel(self._bitmap, x, y, argb)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the pixel colors of the set of pages at the given coordinates.", "response": "def GetPixelColorsHorizontally(self, x: int, y: int, count: int) -> ctypes.Array:\n        \"\"\"\n        x: int.\n        y: int.\n        count: int.\n        Return `ctypes.Array`, an iterable array of int values in argb form point x,y horizontally.\n        \"\"\"\n        arrayType = ctypes.c_uint32 * count\n        values = arrayType()\n        _DllClient.instance().dll.BitmapGetPixelsHorizontally(ctypes.c_size_t(self._bitmap), x, y, values, count)\n        return values"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset the pixel colors form x y horizontally.", "response": "def SetPixelColorsHorizontally(self, x: int, y: int, colors: Iterable) -> bool:\n        \"\"\"\n        Set pixel colors form x,y horizontally.\n        x: int.\n        y: int.\n        colors: Iterable, an iterable list of int color values in argb.\n        Return bool, True if succeed otherwise False.\n        \"\"\"\n        count = len(colors)\n        arrayType = ctypes.c_uint32 * count\n        values = arrayType(*colors)\n        return _DllClient.instance().dll.BitmapSetPixelsHorizontally(ctypes.c_size_t(self._bitmap), x, y, values, count)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef GetPixelColorsVertically(self, x: int, y: int, count: int) -> ctypes.Array:\n        arrayType = ctypes.c_uint32 * count\n        values = arrayType()\n        _DllClient.instance().dll.BitmapGetPixelsVertically(ctypes.c_size_t(self._bitmap), x, y, values, count)\n        return values", "response": "Get the pixel colors at the given point x y vertically."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GetPixelColorsOfRow(self, y: int) -> ctypes.Array:\n        return self.GetPixelColorsOfRect(0, y, self.Width, 1)", "response": "Return an iterable array of int values in argb of y row index."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GetPixelColorsOfColumn(self, x: int) -> ctypes.Array:\n        return self.GetPixelColorsOfRect(x, 0, 1, self.Height)", "response": "Return an iterable array of int values in argb of x column index."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the pixel colors of a rectangle.", "response": "def GetPixelColorsOfRect(self, x: int, y: int, width: int, height: int) -> ctypes.Array:\n        \"\"\"\n        x: int.\n        y: int.\n        width: int.\n        height: int.\n        Return `ctypes.Array`, an iterable array of int values in argb of the input rect.\n        \"\"\"\n        arrayType = ctypes.c_uint32 * (width * height)\n        values = arrayType()\n        _DllClient.instance().dll.BitmapGetPixelsOfRect(ctypes.c_size_t(self._bitmap), x, y, width, height, values)\n        return values"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the pixel colors of the input rect.", "response": "def SetPixelColorsOfRect(self, x: int, y: int, width: int, height: int, colors: Iterable) -> bool:\n        \"\"\"\n        x: int.\n        y: int.\n        width: int.\n        height: int.\n        colors: Iterable, an iterable list of int values, it's length must equal to width*height.\n        Return `ctypes.Array`, an iterable array of int values in argb of the input rect.\n        \"\"\"\n        arrayType = ctypes.c_uint32 * (width * height)\n        values = arrayType(*colors)\n        return bool(_DllClient.instance().dll.BitmapSetPixelsOfRect(ctypes.c_size_t(self._bitmap), x, y, width, height, values))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GetPixelColorsOfRects(self, rects: list) -> list:\n        rects2 = [(x, y, x + width, y + height) for x, y, width, height in rects]\n        left, top, right, bottom = zip(*rects2)\n        left, top, right, bottom = min(left), min(top), max(right), max(bottom)\n        width, height = right - left, bottom - top\n        allColors = self.GetPixelColorsOfRect(left, top, width, height)\n        colorsOfRects = []\n        for x, y, w, h in rects:\n            x -= left\n            y -= top\n            colors = []\n            for row in range(h):\n                colors.extend(allColors[(y + row) * width + x:(y + row) * width + x + w])\n            colorsOfRects.append(colors)\n        return colorsOfRects", "response": "Get the pixel colors of a list of rects."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetAllPixelColors(self) -> ctypes.Array:\n        return self.GetPixelColorsOfRect(0, 0, self.Width, self.Height)", "response": "Returns an iterable array of integer values in argb."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GetSubBitmap(self, x: int, y: int, width: int, height: int) -> 'Bitmap':\n        colors = self.GetPixelColorsOfRect(x, y, width, height)\n        bitmap = Bitmap(width, height)\n        bitmap.SetPixelColorsOfRect(0, 0, width, height, colors)\n        return bitmap", "response": "Get a sub bitmap of the input rect."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Navigate(self, direction: int) -> 'Control':\n        ele = self.pattern.Navigate(direction)\n        return Control.CreateControlFromElement(ele)", "response": "Return the next control in the specified direction within the logical UI tree."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalling IUIAutomationDockPattern::SetDockPosition. dockPosition: int, a value in class `DockPosition`. waitTime: float. Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationdockpattern-setdockposition", "response": "def SetDockPosition(self, dockPosition: int, waitTime: float = OPERATION_WAIT_TIME) -> int:\n        \"\"\"\n        Call IUIAutomationDockPattern::SetDockPosition.\n        dockPosition: int, a value in class `DockPosition`.\n        waitTime: float.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationdockpattern-setdockposition\n        \"\"\"\n        ret = self.pattern.SetDockPosition(dockPosition)\n        time.sleep(waitTime)\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning list of Controls that represent the full set of items that are currently being dragged.", "response": "def GetGrabbedItems(self) -> list:\n        \"\"\"\n        Call IUIAutomationDragPattern::GetCurrentGrabbedItems.\n        Return list, a list of `Control` subclasses that represent the full set of items\n                     that the user is dragging as part of a drag operation.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationdragpattern-getcurrentgrabbeditems\n        \"\"\"\n        eleArray = self.pattern.GetCurrentGrabbedItems()\n        if eleArray:\n            controls = []\n            for i in range(eleArray.Length):\n                ele = eleArray.GetElement(i)\n                con = Control.CreateControlFromElement(element=ele)\n                if con:\n                    controls.append(con)\n            return controls\n        return []"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nexpand the current locale.", "response": "def Expand(self, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        \"\"\"\n        Call IUIAutomationExpandCollapsePattern::Expand.\n        waitTime: float.\n        Return bool, True if succeed otherwise False.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationexpandcollapsepattern-collapse\n        \"\"\"\n        ret = self.pattern.Expand() == S_OK\n        time.sleep(waitTime)\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a control within a containing element based on a specified property value.", "response": "def FindItemByProperty(control: 'Control', propertyId: int, propertyValue) -> 'Control':\n        \"\"\"\n        Call IUIAutomationItemContainerPattern::FindItemByProperty.\n        control: `Control` or its subclass.\n        propertyValue: COM VARIANT according to propertyId? todo.\n        propertyId: int, a value in class `PropertyId`.\n        Return `Control` subclass, a control within a containing element, based on a specified property value.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationitemcontainerpattern-finditembyproperty\n        \"\"\"\n        ele = self.pattern.FindItemByProperty(control.Element, propertyId, propertyValue)\n        return Control.CreateControlFromElement(ele)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GetSelection(self) -> list:\n        eleArray = self.pattern.GetCurrentSelection()\n        if eleArray:\n            controls = []\n            for i in range(eleArray.Length):\n                ele = eleArray.GetElement(i)\n                con = Control.CreateControlFromElement(element=ele)\n                if con:\n                    controls.append(con)\n            return controls\n        return []", "response": "Return a list of Control subclasses that are currently selected."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets the Microsoft Active Accessibility value property for the element.", "response": "def SetValue(self, value: str, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        \"\"\"\n        Call IUIAutomationLegacyIAccessiblePattern::SetValue.\n        Set the Microsoft Active Accessibility value property for the element.\n        value: str.\n        waitTime: float.\n        Return bool, True if succeed otherwise False.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationlegacyiaccessiblepattern-setvalue\n        \"\"\"\n        ret = self.pattern.SetValue(value) == S_OK\n        time.sleep(waitTime)\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef SetView(self, view: int) -> bool:\n        return self.pattern.SetCurrentView(view) == S_OK", "response": "Set the view of the control."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nscrolls the visible region of the content area horizontally and vertically.", "response": "def Scroll(self, horizontalAmount: int, verticalAmount: int, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        \"\"\"\n        Call IUIAutomationScrollPattern::Scroll.\n        Scroll the visible region of the content area horizontally and vertically.\n        horizontalAmount: int, a value in ScrollAmount.\n        verticalAmount: int, a value in ScrollAmount.\n        waitTime: float.\n        Return bool, True if succeed otherwise False.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationscrollpattern-scroll\n        \"\"\"\n        ret = self.pattern.Scroll(horizontalAmount, verticalAmount) == S_OK\n        time.sleep(waitTime)\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef SetScrollPercent(self, horizontalPercent: float, verticalPercent: float, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        ret = self.pattern.SetScrollPercent(horizontalPercent, verticalPercent) == S_OK\n        time.sleep(waitTime)\n        return ret", "response": "Set horizontal and vertical scroll percents of the UI Automation element."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GetAnnotationObjects(self) -> list:\n        eleArray = self.pattern.GetCurrentAnnotationObjects()\n        if eleArray:\n            controls = []\n            for i in range(eleArray.Length):\n                ele = eleArray.GetElement(i)\n                con = Control.CreateControlFromElement(element=ele)\n                if con:\n                    controls.append(con)\n            return controls\n        return []", "response": "Return list of Control subclasses representing the annotations associated with this spreadsheet cell."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the control class that represents the specified name.", "response": "def GetItemByName(self, name: str) -> 'Control':\n        \"\"\"\n        Call IUIAutomationSpreadsheetPattern::GetItemByName.\n        name: str.\n        Return `Control` subclass or None, represents the spreadsheet cell that has the specified name..\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationspreadsheetpattern-getitembyname\n        \"\"\"\n        ele = self.pattern.GetItemByName(name)\n        return Control.CreateControlFromElement(element=ele)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GetColumnHeaderItems(self) -> list:\n        eleArray = self.pattern.GetCurrentColumnHeaderItems()\n        if eleArray:\n            controls = []\n            for i in range(eleArray.Length):\n                ele = eleArray.GetElement(i)\n                con = Control.CreateControlFromElement(element=ele)\n                if con:\n                    controls.append(con)\n            return controls\n        return []", "response": "Return a list of Control subclasses the column headers associated with a table item or cell."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a list of Control subclasses the row headers associated with a table item or cell.", "response": "def GetRowHeaderItems(self) -> list:\n        \"\"\"\n        Call IUIAutomationTableItemPattern::GetCurrentRowHeaderItems.\n        Return list, a list of `Control` subclasses, the row headers associated with a table item or cell.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationtableitempattern-getcurrentrowheaderitems\n        \"\"\"\n        eleArray = self.pattern.GetCurrentRowHeaderItems()\n        if eleArray:\n            controls = []\n            for i in range(eleArray.Length):\n                ele = eleArray.GetElement(i)\n                con = Control.CreateControlFromElement(element=ele)\n                if con:\n                    controls.append(con)\n            return controls\n        return []"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a list of Control subclasses representing all the column headers in a table.", "response": "def GetColumnHeaders(self) -> list:\n        \"\"\"\n        Call IUIAutomationTablePattern::GetCurrentColumnHeaders.\n        Return list, a list of `Control` subclasses, representing all the column headers in a table..\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationtablepattern-getcurrentcolumnheaders\n        \"\"\"\n        eleArray = self.pattern.GetCurrentColumnHeaders()\n        if eleArray:\n            controls = []\n            for i in range(eleArray.Length):\n                ele = eleArray.GetElement(i)\n                con = Control.CreateControlFromElement(element=ele)\n                if con:\n                    controls.append(con)\n            return controls\n        return []"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GetRowHeaders(self) -> list:\n        eleArray = self.pattern.GetCurrentRowHeaders()\n        if eleArray:\n            controls = []\n            for i in range(eleArray.Length):\n                ele = eleArray.GetElement(i)\n                con = Control.CreateControlFromElement(element=ele)\n                if con:\n                    controls.append(con)\n            return controls\n        return []", "response": "Return list of Control subclasses representing all the row headers in a table."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomparing this text range with another.", "response": "def Compare(self, textRange: 'TextRange') -> bool:\n        \"\"\"\n        Call IUIAutomationTextRange::Compare.\n        textRange: `TextRange`.\n        Return bool, specifies whether this text range has the same endpoints as another text range.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationtextrange-compare\n        \"\"\"\n        return bool(self.textRange.Compare(textRange.textRange))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompare the source and target endpoints.", "response": "def CompareEndpoints(self, srcEndPoint: int, textRange: 'TextRange', targetEndPoint: int) -> int:\n        \"\"\"\n        Call IUIAutomationTextRange::CompareEndpoints.\n        srcEndPoint: int, a value in class `TextPatternRangeEndpoint`.\n        textRange: `TextRange`.\n        targetEndPoint: int, a value in class `TextPatternRangeEndpoint`.\n        Return int, a negative value if the caller's endpoint occurs earlier in the text than the target endpoint;\n                    0 if the caller's endpoint is at the same location as the target endpoint;\n                    or a positive value if the caller's endpoint occurs later in the text than the target endpoint.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationtextrange-compareendpoints\n        \"\"\"\n        return self.textRange.CompareEndpoints(srcEndPoint, textRange, targetEndPoint)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn a new TextRange object with the specified text attribute value.", "response": "def FindAttribute(self, textAttributeId: int, val, backward: bool) -> 'TextRange':\n        \"\"\"\n        Call IUIAutomationTextRange::FindAttribute.\n        textAttributeID: int, a value in class `TextAttributeId`.\n        val: COM VARIANT according to textAttributeId? todo.\n        backward: bool, True if the last occurring text range should be returned instead of the first; otherwise False.\n        return `TextRange` or None, a text range subset that has the specified text attribute value.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationtextrange-findattribute\n        \"\"\"\n        textRange = self.textRange.FindAttribute(textAttributeId, val, int(backward))\n        if textRange:\n            return TextRange(textRange=textRange)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a TextRange object that contains the specified text.", "response": "def FindText(self, text: str, backward: bool, ignoreCase: bool) -> 'TextRange':\n        \"\"\"\n        Call IUIAutomationTextRange::FindText.\n        text: str,\n        backward: bool, True if the last occurring text range should be returned instead of the first; otherwise False.\n        ignoreCase: bool, True if case should be ignored; otherwise False.\n        return `TextRange` or None, a text range subset that contains the specified text.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationtextrange-findtext\n        \"\"\"\n        textRange = self.textRange.FindText(text, int(backward), int(ignoreCase))\n        if textRange:\n            return TextRange(textRange=textRange)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncall IUIAutomationTextRange :: GetAttributeValue.", "response": "def GetAttributeValue(self, textAttributeId: int) -> ctypes.POINTER(comtypes.IUnknown):\n        \"\"\"\n        Call IUIAutomationTextRange::GetAttributeValue.\n        textAttributeId: int, a value in class `TextAttributeId`.\n        Return `ctypes.POINTER(comtypes.IUnknown)` or None, the value of the specified text attribute across the entire text range, todo.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationtextrange-getattributevalue\n        \"\"\"\n        return self.textRange.GetAttributeValue(textAttributeId)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GetBoundingRectangles(self) -> list:\n        floats = self.textRange.GetBoundingRectangles()\n        rects = []\n        for i in range(len(floats) // 4):\n            rect = Rect(int(floats[i * 4]), int(floats[i * 4 + 1]),\n                                        int(floats[i * 4]) + int(floats[i * 4 + 2]), int(floats[i * 4 + 1]) + int(floats[i * 4 + 3]))\n            rects.append(rect)\n        return rects", "response": "Return list of rectangles for each fully visible line of text in a text range."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetChildren(self) -> list:\n        eleArray = self.textRange.GetChildren()\n        if eleArray:\n            controls = []\n            for i in range(eleArray.Length):\n                ele = eleArray.GetElement(i)\n                con = Control.CreateControlFromElement(element=ele)\n                if con:\n                    controls.append(con)\n            return controls\n        return []", "response": "Call IUIAutomationTextRange :: GetChildren."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Move(self, unit: int, count: int, waitTime: float = OPERATION_WAIT_TIME) -> int:\n        ret = self.textRange.Move(unit, count)\n        time.sleep(waitTime)\n        return ret", "response": "Move the text range forward or backward by the specified number of text units."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MoveEndpointByRange(self, srcEndPoint: int, textRange: 'TextRange', targetEndPoint: int, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        ret = self.textRange.MoveEndpointByRange(srcEndPoint, textRange.textRange, targetEndPoint) == S_OK\n        time.sleep(waitTime)\n        return ret", "response": "Move one endpoint of the current text range to another."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef MoveEndpointByUnit(self, endPoint: int, unit: int, count: int, waitTime: float = OPERATION_WAIT_TIME) -> int:\n        ret = self.textRange.MoveEndpointByUnit(endPoint, unit, count)\n        time.sleep(waitTime)\n        return ret", "response": "Move one endpoint of the text pattern range by unit."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nscrolls the text range into view.", "response": "def ScrollIntoView(self, alignTop: bool = True, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        \"\"\"\n        Call IUIAutomationTextRange::ScrollIntoView.\n        Cause the text control to scroll until the text range is visible in the viewport.\n        alignTop: bool, True if the text control should be scrolled so that the text range is flush with the top of the viewport;\n                        False if it should be flush with the bottom of the viewport.\n        waitTime: float.\n        Return bool, True if succeed otherwise False.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationtextrange-scrollintoview\n        \"\"\"\n        ret = self.textRange.ScrollIntoView(int(alignTop)) == S_OK\n        time.sleep(waitTime)\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GetActiveComposition(self) -> TextRange:\n        textRange = self.pattern.GetActiveComposition()\n        if textRange:\n            return TextRange(textRange=textRange)", "response": "Return the active composition."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GetConversionTarget(self) -> TextRange:\n        textRange = self.pattern.GetConversionTarget()\n        if textRange:\n            return TextRange(textRange=textRange)", "response": "Return the current conversion target range."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a list of TextRange disjoint text ranges from a text - based control where each text range represents a contiguous span of visible text.", "response": "def GetVisibleRanges(self) -> list:\n        \"\"\"\n        Call IUIAutomationTextPattern::GetVisibleRanges.\n        Return list, a list of `TextRange`, disjoint text ranges from a text-based control\n                     where each text range represents a contiguous span of visible text.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationtextpattern-getvisibleranges\n        \"\"\"\n        eleArray = self.pattern.GetVisibleRanges()\n        if eleArray:\n            textRanges = []\n            for i in range(eleArray.Length):\n                ele = eleArray.GetElement(i)\n                textRanges.append(TextRange(textRange=ele))\n            return textRanges\n        return []"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a TextRange object representing the text range of the child element.", "response": "def RangeFromChild(self, child) -> TextRange:\n        \"\"\"\n        Call IUIAutomationTextPattern::RangeFromChild.\n        child: `Control` or its subclass.\n        Return `TextRange` or None, a text range enclosing a child element such as an image,\n            hyperlink, Microsoft Excel spreadsheet, or other embedded object.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationtextpattern-rangefromchild\n        \"\"\"\n        textRange = self.pattern.RangeFromChild(Control.Element)\n        if textRange:\n            return TextRange(textRange=textRange)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef RangeFromPoint(self, x: int, y: int) -> TextRange:\n        textRange = self.pattern.RangeFromPoint(ctypes.wintypes.POINT(x, y))\n        if textRange:\n            return TextRange(textRange=textRange)", "response": "Return a new TextRange object for the specified screen coordinates."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Move(self, x: int, y: int, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        ret = self.pattern.Move(x, y) == S_OK\n        time.sleep(waitTime)\n        return ret", "response": "Move the UI Automation element."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Resize(self, width: int, height: int, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        ret = self.pattern.Resize(width, height) == S_OK\n        time.sleep(waitTime)\n        return ret", "response": "Resize the UI Automation element."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Rotate(self, degrees: int, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        ret = self.pattern.Rotate(degrees) == S_OK\n        time.sleep(waitTime)\n        return ret", "response": "Rotate the UI Automation element."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Zoom(self, zoomLevel: float, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        ret = self.pattern.Zoom(zoomLevel) == S_OK\n        time.sleep(waitTime)\n        return ret", "response": "Zoom the viewport of the control."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ZoomByUnit(self, zoomUnit: int, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        ret = self.pattern.ZoomByUnit(zoomUnit) == S_OK\n        time.sleep(waitTime)\n        return ret", "response": "Zoom the viewport of the control by the specified unit."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets the state of the window.", "response": "def SetWindowVisualState(self, state: int, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        \"\"\"\n        Call IUIAutomationWindowPattern::SetWindowVisualState.\n        Minimize, maximize, or restore the window.\n        state: int, a value in class `WindowVisualState`.\n        waitTime: float.\n        Return bool, True if succeed otherwise False.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationwindowpattern-setwindowvisualstate\n        \"\"\"\n        ret = self.pattern.SetWindowVisualState(state) == S_OK\n        time.sleep(waitTime)\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalling IUIAutomationWindowPattern::WaitForInputIdle. Cause the calling code to block for the specified time or until the associated process enters an idle state, whichever completes first. milliseconds: int. Return bool, True if succeed otherwise False. Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationwindowpattern-waitforinputidle", "response": "def WaitForInputIdle(self, milliseconds: int) -> bool:\n        '''\n        Call IUIAutomationWindowPattern::WaitForInputIdle.\n        Cause the calling code to block for the specified time or\n            until the associated process enters an idle state, whichever completes first.\n        milliseconds: int.\n        Return bool, True if succeed otherwise False.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationwindowpattern-waitforinputidle\n        '''\n        return self.pattern.WaitForInputIdle(milliseconds) == S_OK"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate a concreate control from an element.", "response": "def CreateControlFromElement(element) -> 'Control':\n        \"\"\"\n        Create a concreate `Control` from a com type `IUIAutomationElement`.\n        element: `ctypes.POINTER(IUIAutomationElement)`.\n        Return a subclass of `Control`, an instance of the control's real type.\n        \"\"\"\n        if element:\n            controlType = element.CurrentControlType\n            if controlType in ControlConstructors:\n                return ControlConstructors[controlType](element=element)\n            else:\n                Logger.WriteLine(\"element.CurrentControlType returns {}, invalid ControlType!\".format(controlType), ConsoleColor.Red)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding search properties using dict. update.", "response": "def AddSearchProperties(self, **searchProperties) -> None:\n        \"\"\"\n        Add search properties using `dict.update`.\n        searchProperties: dict, same as searchProperties in `Control.__init__`.\n        \"\"\"\n        self.searchProperties.update(searchProperties)\n        if 'Depth' in searchProperties:\n            self.searchDepth = searchProperties['Depth']\n        if 'RegexName' in searchProperties:\n            regName = searchProperties['RegexName']\n            self.regexName = re.compile(regName) if regName else None"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nremoving the specified search properties from the instance.", "response": "def RemoveSearchProperties(self, **searchProperties) -> None:\n        \"\"\"\n        searchProperties: dict, same as searchProperties in `Control.__init__`.\n        \"\"\"\n        for key in searchProperties:\n            del self.searchProperties[key]\n            if key == 'RegexName':\n                self.regexName = None"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GetColorfulSearchPropertiesStr(self, keyColor='DarkGreen', valueColor='DarkCyan') -> str:\n        strs = ['<Color={}>{}</Color>: <Color={}>{}</Color>'.format(keyColor if k in Control.ValidKeys else 'DarkYellow', k, valueColor,\n                ControlTypeNames[v] if k == 'ControlType' else repr(v)) for k, v in self.searchProperties.items()]\n        return '{' + ', '.join(strs) + '}'", "response": "Get a string representation of the search properties."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef BoundingRectangle(self) -> Rect:\n        rect = self.Element.CurrentBoundingRectangle\n        return Rect(rect.left, rect.top, rect.right, rect.bottom)", "response": "Return the current bounding rectangle of the element."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncall IUIAutomationElement::GetClickablePoint. Return tuple, (x: int, y: int, gotClickable: bool), like (20, 10, True) Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationelement-getclickablepoint", "response": "def GetClickablePoint(self) -> tuple:\n        \"\"\"\n        Call IUIAutomationElement::GetClickablePoint.\n        Return tuple, (x: int, y: int, gotClickable: bool), like (20, 10, True)\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationelement-getclickablepoint\n        \"\"\"\n        point, gotClickable = self.Element.GetClickablePoint()\n        return (point.x, point.y, bool(gotClickable))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalling IUIAutomationElement :: GetCurrentPattern. Get a new pattern by pattern id if it supports the pattern.", "response": "def GetPattern(self, patternId: int):\n        \"\"\"\n        Call IUIAutomationElement::GetCurrentPattern.\n        Get a new pattern by pattern id if it supports the pattern.\n        patternId: int, a value in class `PatternId`.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationelement-getcurrentpattern\n        \"\"\"\n        try:\n            pattern = self.Element.GetCurrentPattern(patternId)\n            if pattern:\n                subPattern = CreatePattern(patternId, pattern)\n                self._supportedPatterns[patternId] = subPattern\n                return subPattern\n        except comtypes.COMError as ex:\n            pass"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef GetPatternAs(self, patternId: int, riid):\n        return self.Element.GetCurrentPatternAs(patternId, riid)", "response": "Call IUIAutomationElement::GetCurrentPatternAs.\n        Get a new pattern by pattern id if it supports the pattern, todo.\n        patternId: int, a value in class `PatternId`.\n        riid: GUID.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationelement-getcurrentpatternas"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting the value of the property with the given id.", "response": "def GetPropertyValueEx(self, propertyId: int, ignoreDefaultValue: int) -> Any:\n        \"\"\"\n        Call IUIAutomationElement::GetCurrentPropertyValueEx.\n        propertyId: int, a value in class `PropertyId`.\n        ignoreDefaultValue: int, 0 or 1.\n        Return Any, corresponding type according to propertyId.\n        Refer https://docs.microsoft.com/en-us/windows/desktop/api/uiautomationclient/nf-uiautomationclient-iuiautomationelement-getcurrentpropertyvalueex\n        \"\"\"\n        return self.Element.GetCurrentPropertyValueEx(propertyId, ignoreDefaultValue)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GetCachedPattern(self, patternId: int, cache: bool):\n        if cache:\n            pattern = self._supportedPatterns.get(patternId, None)\n            if pattern:\n                return pattern\n            else:\n                pattern = self.GetPattern(patternId)\n                if pattern:\n                    self._supportedPatterns[patternId] = pattern\n                    return pattern\n        else:\n            pattern = self.GetPattern(patternId)\n            if pattern:\n                self._supportedPatterns[patternId] = pattern\n                return pattern", "response": "Get a pattern by patternId."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets an ancestor control that matches the condition.", "response": "def GetAncestorControl(self, condition: Callable) -> 'Control':\n        \"\"\"\n        Get a ancestor control that matches the condition.\n        condition: Callable, function (control: Control, depth: int)->bool,\n                   depth starts with -1 and decreses when search goes up.\n        Return `Control` subclass or None.\n        \"\"\"\n        ancestor = self\n        depth = 0\n        while True:\n            ancestor = ancestor.GetParentControl()\n            depth -= 1\n            if ancestor:\n                if condition(ancestor, depth):\n                    return ancestor\n            else:\n                break"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the parent control.", "response": "def GetParentControl(self) -> 'Control':\n        \"\"\"\n        Return `Control` subclass or None.\n        \"\"\"\n        ele = _AutomationClient.instance().ViewWalker.GetParentElement(self.Element)\n        return Control.CreateControlFromElement(ele)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef GetFirstChildControl(self) -> 'Control':\n        ele = _AutomationClient.instance().ViewWalker.GetFirstChildElement(self.Element)\n        return Control.CreateControlFromElement(ele)", "response": "Get the first child control of this view."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GetLastChildControl(self) -> 'Control':\n        ele = _AutomationClient.instance().ViewWalker.GetLastChildElement(self.Element)\n        return Control.CreateControlFromElement(ele)", "response": "Get the last child control."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef GetNextSiblingControl(self) -> 'Control':\n        ele = _AutomationClient.instance().ViewWalker.GetNextSiblingElement(self.Element)\n        return Control.CreateControlFromElement(ele)", "response": "Get the next sibling control."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the previous sibling control.", "response": "def GetPreviousSiblingControl(self) -> 'Control':\n        \"\"\"\n        Return `Control` subclass or None.\n        \"\"\"\n        ele = _AutomationClient.instance().ViewWalker.GetPreviousSiblingElement(self.Element)\n        return Control.CreateControlFromElement(ele)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef GetSiblingControl(self, condition: Callable, forward: bool = True) -> 'Control':\n        if not forward:\n            prev = self\n            while True:\n                prev = prev.GetPreviousSiblingControl()\n                if prev:\n                    if condition(prev):\n                        return prev\n                else:\n                    break\n        next_ = self\n        while True:\n            next_ = next_.GetNextSiblingControl()\n            if next_:\n                if condition(next_):\n                    return next_\n            else:\n                break", "response": "Returns a SiblingControl subclass or None."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns list of Control subclasses.", "response": "def GetChildren(self) -> list:\n        \"\"\"\n        Return list, a list of `Control` subclasses.\n        \"\"\"\n        children = []\n        child = self.GetFirstChildControl()\n        while child:\n            children.append(child)\n            child = child.GetNextSiblingControl()\n        return children"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _CompareFunction(self, control: 'Control', depth: int) -> bool:\n        for key, value in self.searchProperties.items():\n            if 'ControlType' == key:\n                if value != control.ControlType:\n                    return False\n            elif 'ClassName' == key:\n                if value != control.ClassName:\n                    return False\n            elif 'AutomationId' == key:\n                if value != control.AutomationId:\n                    return False\n            elif 'Name' == key:\n                if value != control.Name:\n                    return False\n            elif 'SubName' == key:\n                if value not in control.Name:\n                    return False\n            elif 'RegexName' == key:\n                if not self.regexName.match(control.Name):\n                    return False\n            elif 'Depth' == key:\n                if value != depth:\n                    return False\n            elif 'Compare' == key:\n                if not value(control, depth):\n                    return False\n        return True", "response": "Return True if the control or sub - class matches the tree depth."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncheck if a specific element or control exists.", "response": "def Exists(self, maxSearchSeconds: float = 5, searchIntervalSeconds: float = SEARCH_INTERVAL, printIfNotExist: bool = False) -> bool:\n        \"\"\"\n        maxSearchSeconds: float\n        searchIntervalSeconds: float\n        Find control every searchIntervalSeconds seconds in maxSearchSeconds seconds.\n        Return bool, True if find\n        \"\"\"\n        if self._element and self._elementDirectAssign:\n            #if element is directly assigned, not by searching, just check whether self._element is valid\n            #but I can't find an API in UIAutomation that can directly check\n            rootElement = GetRootControl().Element\n            if self._element == rootElement:\n                return True\n            else:\n                parentElement = _AutomationClient.instance().ViewWalker.GetParentElement(self._element)\n                if parentElement:\n                    return True\n                else:\n                    return False\n        #find the element\n        if len(self.searchProperties) == 0:\n            raise LookupError(\"control's searchProperties must not be empty!\")\n        self._element = None\n        startTime = ProcessTime()\n        # Use same timeout(s) parameters for resolve all parents\n        prev =  self.searchFromControl\n        if prev and not prev._element and not prev.Exists(maxSearchSeconds, searchIntervalSeconds):\n            if printIfNotExist or DEBUG_EXIST_DISAPPEAR:\n                Logger.ColorfullyWriteLine(self.GetColorfulSearchPropertiesStr() + '<Color=Red> does not exist.</Color>')\n            return False\n        startTime2 = ProcessTime()\n        if DEBUG_SEARCH_TIME:\n            startDateTime = datetime.datetime.now()\n        while True:\n            control = FindControl(self.searchFromControl, self._CompareFunction, self.searchDepth, False, self.foundIndex)\n            if control:\n                self._element = control.Element\n                control._element = 0  # control will be destroyed, but the element needs to be stroed in self._element\n                if DEBUG_SEARCH_TIME:\n                    Logger.ColorfullyWriteLine('{} TraverseControls: <Color=Cyan>{}</Color>, SearchTime: <Color=Cyan>{:.3f}</Color>s[{} - {}]'.format(\n                        self.GetColorfulSearchPropertiesStr(), control.traverseCount, ProcessTime() - startTime2,\n                        startDateTime.time(), datetime.datetime.now().time()))\n                return True\n            else:\n                remain = startTime + maxSearchSeconds - ProcessTime()\n                if remain > 0:\n                    time.sleep(min(remain, searchIntervalSeconds))\n                else:\n                    if printIfNotExist or DEBUG_EXIST_DISAPPEAR:\n                        Logger.ColorfullyWriteLine(self.GetColorfulSearchPropertiesStr() + '<Color=Red> does not exist.</Color>')\n                    return False"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef Disappears(self, maxSearchSeconds: float = 5, searchIntervalSeconds: float = SEARCH_INTERVAL, printIfNotDisappear: bool = False) -> bool:\n        global DEBUG_EXIST_DISAPPEAR\n        start = ProcessTime()\n        while True:\n            temp = DEBUG_EXIST_DISAPPEAR\n            DEBUG_EXIST_DISAPPEAR = False  # do not print for Exists\n            if not self.Exists(0, 0, False):\n                DEBUG_EXIST_DISAPPEAR = temp\n                return True\n            DEBUG_EXIST_DISAPPEAR = temp\n            remain = start + maxSearchSeconds - ProcessTime()\n            if remain > 0:\n                time.sleep(min(remain, searchIntervalSeconds))\n            else:\n                if printIfNotDisappear or DEBUG_EXIST_DISAPPEAR:\n                    Logger.ColorfullyWriteLine(self.GetColorfulSearchPropertiesStr() + '<Color=Red> does not disappear.</Color>')\n                return False", "response": "Check if the current object is disappears every searchIntervalSeconds seconds."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef MoveCursorToInnerPos(self, x: int = None, y: int = None, ratioX: float = 0.5, ratioY: float = 0.5, simulateMove: bool = True) -> tuple:\n        rect = self.BoundingRectangle\n        if rect.width() == 0 or rect.height() == 0:\n            Logger.ColorfullyWriteLine('<Color=Yellow>Can not move curosr</Color>. {}\\'s BoundingRectangle is {}. SearchProperties: {}'.format(\n                self.ControlTypeName, rect, self.GetColorfulSearchPropertiesStr()))\n            return\n        if x is None:\n            x = rect.left + int(rect.width() * ratioX)\n        else:\n            x = (rect.left if x >= 0 else rect.right) + x\n        if y is None:\n            y = rect.top + int(rect.height() * ratioY)\n        else:\n            y = (rect.top if y >= 0 else rect.bottom) + y\n        if simulateMove and MAX_MOVE_SECOND > 0:\n            MoveTo(x, y, waitTime=0)\n        else:\n            SetCursorPos(x, y)\n        return x, y", "response": "Moves the cursor to the inner position of the control."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmoving the cursor to the control s center.", "response": "def MoveCursorToMyCenter(self, simulateMove: bool = True) -> tuple:\n        \"\"\"\n        Move cursor to control's center.\n        Return tuple, two ints tuple(x,y), the cursor positon relative to screen(0,0) after moving .\n        \"\"\"\n        return self.MoveCursorToInnerPos(simulateMove=simulateMove)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef Click(self, x: int = None, y: int = None, ratioX: float = 0.5, ratioY: float = 0.5, simulateMove: bool = True, waitTime: float = OPERATION_WAIT_TIME) -> None:\n        point = self.MoveCursorToInnerPos(x, y, ratioX, ratioY, simulateMove)\n        if point:\n            Click(point[0], point[1], waitTime)", "response": "Clicks the specified location."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndoubling click the current entry.", "response": "def DoubleClick(self, x: int = None, y: int = None, ratioX: float = 0.5, ratioY: float = 0.5, simulateMove: bool = True, waitTime: float = OPERATION_WAIT_TIME) -> None:\n        \"\"\"\n        x: int, if < 0, right click self.BoundingRectangle.right + x, if not None, ignore ratioX.\n        y: int, if < 0, right click self.BoundingRectangle.bottom + y, if not None, ignore ratioY.\n        ratioX: float.\n        ratioY: float.\n        simulateMove: bool, if True, first move cursor to control smoothly.\n        waitTime: float.\n\n        DoubleClick(), DoubleClick(ratioX=0.5, ratioY=0.5): double click center.\n        DoubleClick(10, 10): double click left+10, top+10.\n        DoubleClick(-10, -10): double click right-10, bottom-10.\n        \"\"\"\n        x, y = self.MoveCursorToInnerPos(x, y, ratioX, ratioY, simulateMove)\n        Click(x, y, GetDoubleClickTime() * 1.0 / 2000)\n        Click(x, y, waitTime)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nmakes control have focus first move cursor to center and mouse wheel down.", "response": "def WheelDown(self, wheelTimes: int = 1, interval: float = 0.05, waitTime: float = OPERATION_WAIT_TIME) -> None:\n        \"\"\"\n        Make control have focus first, move cursor to center and mouse wheel down.\n        wheelTimes: int.\n        interval: float.\n        waitTime: float.\n        \"\"\"\n        x, y = GetCursorPos()\n        self.SetFocus()\n        self.MoveCursorToMyCenter(False)\n        WheelDown(wheelTimes, interval, waitTime)\n        SetCursorPos(x, y)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nshow a native window.", "response": "def ShowWindow(self, cmdShow: int, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        \"\"\"\n        Get a native handle from self or ancestors until valid and call native `ShowWindow` with cmdShow.\n        cmdShow: int, a value in in class `SW`.\n        waitTime: float.\n        Return bool, True if succeed otherwise False.\n        \"\"\"\n        handle = self.NativeWindowHandle\n        if not handle:\n            control = self\n            while not handle:\n                control = control.GetParentControl()\n                handle = control.NativeWindowHandle\n        if handle:\n            ret = ShowWindow(handle, cmdShow)\n            time.sleep(waitTime)\n            return ret"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nshow the current window.", "response": "def Show(self, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        \"\"\"\n        Call native `ShowWindow(SW.Show)`.\n        Return bool, True if succeed otherwise False.\n        \"\"\"\n        return self.ShowWindow(SW.Show, waitTime)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncalls native `ShowWindow(SW.Hide)`. waitTime: float Return bool, True if succeed otherwise False.", "response": "def Hide(self, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        \"\"\"\n        Call native `ShowWindow(SW.Hide)`.\n        waitTime: float\n        Return bool, True if succeed otherwise False.\n        \"\"\"\n        return self.ShowWindow(SW.Hide, waitTime)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalling native MoveWindow if control has a valid native handle. x: int. y: int. width: int. height: int. repaint: bool. Return bool, True if succeed otherwise False.", "response": "def MoveWindow(self, x: int, y: int, width: int, height: int, repaint: bool = True) -> bool:\n        \"\"\"\n        Call native MoveWindow if control has a valid native handle.\n        x: int.\n        y: int.\n        width: int.\n        height: int.\n        repaint: bool.\n        Return bool, True if succeed otherwise False.\n        \"\"\"\n        handle = self.NativeWindowHandle\n        if handle:\n            return MoveWindow(handle, x, y, width, height, int(repaint))\n        return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef SetWindowText(self, text: str) -> bool:\n        handle = self.NativeWindowHandle\n        if handle:\n            return SetWindowText(handle, text)\n        return False", "response": "Set the text of the current window."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsends a key to the control.", "response": "def SendKey(self, key: int, waitTime: float = OPERATION_WAIT_TIME) -> None:\n        \"\"\"\n        Make control have focus first and type a key.\n        `self.SetFocus` may not work for some controls, you may need to click it to make it have focus.\n        key: int, a key code value in class Keys.\n        waitTime: float.\n        \"\"\"\n        self.SetFocus()\n        SendKey(key, waitTime)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef SendKeys(self, keys: str, interval: float = 0.01, waitTime: float = OPERATION_WAIT_TIME) -> None:\n        self.SetFocus()\n        SendKeys(keys, interval, waitTime)", "response": "Send a set of keys to the control."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the color of the specified pixel.", "response": "def GetPixelColor(self, x: int, y: int) -> int:\n        \"\"\"\n        Call native `GetPixelColor` if control has a valid native handle.\n        Use `self.ToBitmap` if control doesn't have a valid native handle or you get many pixels.\n        x: int, internal x position.\n        y: int, internal y position.\n        Return int, a color value in bgr.\n        r = bgr & 0x0000FF\n        g = (bgr & 0x00FF00) >> 8\n        b = (bgr & 0xFF0000) >> 16\n        \"\"\"\n        handle = self.NativeWindowHandle\n        if handle:\n            return GetPixelColor(x, y, handle)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert a CaptureControl object to a Bitmap object.", "response": "def ToBitmap(self, x: int = 0, y: int = 0, width: int = 0, height: int = 0) -> Bitmap:\n        \"\"\"\n        Capture control to a Bitmap object.\n        x, y: int, the point in control's internal position(from 0,0).\n        width, height: int, image's width and height from x, y, use 0 for entire area.\n                       If width(or height) < 0, image size will be control's width(or height) - width(or height).\n        \"\"\"\n        bitmap = Bitmap()\n        bitmap.FromControl(self, x, y, width, height)\n        return bitmap"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncaptures control to an image file.", "response": "def CaptureToImage(self, savePath: str, x: int = 0, y: int = 0, width: int = 0, height: int = 0) -> bool:\n        \"\"\"\n        Capture control to a image file.\n        savePath: str, should end with .bmp, .jpg, .jpeg, .png, .gif, .tif, .tiff.\n        x, y: int, the point in control's internal position(from 0,0).\n        width, height: int, image's width and height from x, y, use 0 for entire area.\n                       If width(or height) < 0, image size will be control's width(or height) - width(or height).\n        Return bool, True if succeed otherwise False.\n        \"\"\"\n        bitmap = Bitmap()\n        if bitmap.FromControl(self, x, y, width, height):\n            return bitmap.ToFile(savePath)\n        return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef IsTopLevel(self) -> bool:\n        handle = self.NativeWindowHandle\n        if handle:\n            return GetAncestor(handle, GAFlag.Root) == handle\n        return False", "response": "Determines whether the current control is top level."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the top level control which current control lays.", "response": "def GetTopLevelControl(self) -> 'Control':\n        \"\"\"\n        Get the top level control which current control lays.\n        If current control is top level, return self.\n        If current control is root control, return None.\n        Return `PaneControl` or `WindowControl` or None.\n        \"\"\"\n        handle = self.NativeWindowHandle\n        if handle:\n            topHandle = GetAncestor(handle, GAFlag.Root)\n            if topHandle:\n                if topHandle == handle:\n                    return self\n                else:\n                    return ControlFromHandle(topHandle)\n            else:\n                #self is root control\n                pass\n        else:\n            control = self\n            while True:\n                control = control.GetParentControl()\n                handle = control.NativeWindowHandle\n                if handle:\n                    topHandle = GetAncestor(handle, GAFlag.Root)\n                    return ControlFromHandle(topHandle)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Select(self, itemName: str = '', condition: Callable = None, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        expandCollapsePattern = self.GetExpandCollapsePattern()\n        if expandCollapsePattern:\n            expandCollapsePattern.Expand()\n        else:\n            #Windows Form's ComboBoxControl doesn't support ExpandCollapsePattern\n            self.Click(x=-10, ratioY=0.5, simulateMove=False)\n        find = False\n        if condition:\n            listItemControl = self.ListItemControl(Compare=lambda c, d: condition(c.Name))\n        else:\n            listItemControl = self.ListItemControl(Name=itemName)\n        if listItemControl.Exists(1):\n            scrollItemPattern = listItemControl.GetScrollItemPattern()\n            if scrollItemPattern:\n                scrollItemPattern.ScrollIntoView(waitTime=0.1)\n            listItemControl.Click(waitTime=waitTime)\n            find = True\n        else:\n            #ComboBox's popup window is a child of root control\n            listControl = ListControl(searchDepth= 1)\n            if listControl.Exists(1):\n                if condition:\n                    listItemControl = self.ListItemControl(Compare=lambda c, d: condition(c.Name))\n                else:\n                    listItemControl = self.ListItemControl(Name=itemName)\n                if listItemControl.Exists(0, 0):\n                    scrollItemPattern = listItemControl.GetScrollItemPattern()\n                    if scrollItemPattern:\n                        scrollItemPattern.ScrollIntoView(waitTime=0.1)\n                    listItemControl.Click(waitTime=waitTime)\n                    find = True\n        if not find:\n            Logger.ColorfullyWriteLine('Can\\'t find <Color=Cyan>{}</Color> in ComboBoxControl or it does not support selection.'.format(itemName), ConsoleColor.Yellow)\n            if expandCollapsePattern:\n                expandCollapsePattern.Collapse(waitTime)\n            else:\n                self.Click(x=-10, ratioY=0.5, simulateMove=False, waitTime=waitTime)\n        return find", "response": "Show combobox s popup menu and select a item by name."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets top level window topmost.", "response": "def SetTopmost(self, isTopmost: bool = True, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        \"\"\"\n        Set top level window topmost.\n        isTopmost: bool.\n        waitTime: float.\n        \"\"\"\n        if self.IsTopLevel():\n            ret = SetWindowTopmost(self.NativeWindowHandle, isTopmost)\n            time.sleep(waitTime)\n            return ret\n        return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef Maximize(self, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        if self.IsTopLevel():\n            return self.ShowWindow(SW.ShowMaximized, waitTime)\n        return False", "response": "Maximize the current window."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MoveToCenter(self) -> bool:\n        if self.IsTopLevel():\n            rect = self.BoundingRectangle\n            screenWidth, screenHeight = GetScreenSize()\n            x, y = (screenWidth - rect.width()) // 2, (screenHeight - rect.height()) // 2\n            if x < 0: x = 0\n            if y < 0: y = 0\n            return SetWindowPos(self.NativeWindowHandle, SWP.HWND_Top, x, y, 0, 0, SWP.SWP_NoSize)\n        return False", "response": "Moves the window to the screen center."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef SetActive(self, waitTime: float = OPERATION_WAIT_TIME) -> bool:\n        if self.IsTopLevel():\n            handle = self.NativeWindowHandle\n            if IsIconic(handle):\n                ret = ShowWindow(handle, SW.Restore)\n            elif not IsWindowVisible(handle):\n                ret = ShowWindow(handle, SW.Show)\n            ret = SetForegroundWindow(handle)  # may fail if foreground windows's process is not python\n            time.sleep(waitTime)\n            return ret\n        return False", "response": "Set top level window active."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef MetroClose(self, waitTime: float = OPERATION_WAIT_TIME) -> None:\n        if self.ClassName == METRO_WINDOW_CLASS_NAME:\n            screenWidth, screenHeight = GetScreenSize()\n            MoveTo(screenWidth // 2, 0, waitTime=0)\n            DragDrop(screenWidth // 2, 0, screenWidth // 2, screenHeight, waitTime=waitTime)\n        else:\n            Logger.WriteLine('Window is not Metro!', ConsoleColor.Yellow)", "response": "Only work on Windows 8/8.1, if current window is Metro UI.\n        waitTime: float."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef DemoCN():\n    thisWindow = auto.GetConsoleWindow()\n    auto.Logger.ColorfullyWrite('\u6211\u5c06\u8fd0\u884c<Color=Cyan>cmd</Color>\u5e76\u8bbe\u7f6e\u5b83\u7684<Color=Cyan>\u5c4f\u5e55\u7f13\u51b2\u533a</Color>\u4f7f<Color=Cyan>cmd</Color>\u4e00\u884c\u80fd\u5bb9\u7eb3\u5f88\u591a\u5b57\u7b26\\n\\n')\n    time.sleep(3)\n\n    auto.SendKeys('{Win}r')\n    while not isinstance(auto.GetFocusedControl(), auto.EditControl):\n        time.sleep(1)\n    auto.SendKeys('cmd{Enter}')\n    cmdWindow = auto.WindowControl(RegexName = '.+cmd.exe')\n    cmdWindow.TitleBarControl().RightClick()\n    auto.SendKey(auto.Keys.VK_P)\n    optionWindow = cmdWindow.WindowControl(SubName = '\u5c5e\u6027')\n    optionWindow.TabItemControl(SubName = '\u9009\u9879').Click()\n    optionTab = optionWindow.PaneControl(SubName = '\u9009\u9879')\n    checkBox = optionTab.CheckBoxControl(AutomationId = '103')\n    if checkBox.GetTogglePattern().ToggleState != auto.ToggleState.On:\n        checkBox.Click()\n    checkBox = optionTab.CheckBoxControl(AutomationId = '104')\n    if checkBox.GetTogglePattern().ToggleState != auto.ToggleState.On:\n        checkBox.Click()\n    optionWindow.TabItemControl(SubName = '\u5e03\u5c40').Click()\n    layoutTab = optionWindow.PaneControl(SubName = '\u5e03\u5c40')\n    layoutTab.EditControl(AutomationId='301').GetValuePattern().SetValue('300')\n    layoutTab.EditControl(AutomationId='303').GetValuePattern().SetValue('3000')\n    layoutTab.EditControl(AutomationId='305').GetValuePattern().SetValue('140')\n    layoutTab.EditControl(AutomationId='307').GetValuePattern().SetValue('30')\n    optionWindow.ButtonControl(AutomationId = '1').Click()\n    cmdWindow.SetActive()\n    rect = cmdWindow.BoundingRectangle\n    auto.DragDrop(rect.left + 50, rect.top + 10, 50, 30)\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('\u6211\u5c06\u8fd0\u884c<Color=Cyan>\u8bb0\u4e8b\u672c</Color>\u5e76\u8f93\u5165<Color=Cyan>Hello!!!</Color>\\n\\n')\n    time.sleep(3)\n\n    subprocess.Popen('notepad')\n    notepadWindow = auto.WindowControl(searchDepth = 1, ClassName = 'Notepad')\n    cx, cy = auto.GetScreenSize()\n    notepadWindow.MoveWindow(cx // 2, 20, cx // 2, cy // 2)\n    time.sleep(0.5)\n    notepadWindow.EditControl().SendKeys('Hello!!!', 0.05)\n    time.sleep(1)\n\n    dir = os.path.dirname(__file__)\n    scriptPath = os.path.abspath(os.path.join(dir, '..\\\\automation.py'))\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('\u8fd0\u884c\"<Color=Cyan>automation.py -h</Color>\"\u663e\u793a\u5e2e\u52a9\\n\\n')\n    time.sleep(3)\n\n    cmdWindow.SendKeys('\"{}\" -h'.format(scriptPath) + '{Enter}', 0.05)\n    time.sleep(3)\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('\u8fd0\u884c\"<Color=Cyan>automation.py -r -d1</Color>\"\u663e\u793a\u6240\u6709\u9876\u5c42\u7a97\u53e3, \u5373\u684c\u9762\u7684\u5b50\u7a97\u53e3\\n\\n')\n    time.sleep(3)\n\n    cmdWindow.SendKeys('\"{}\" -r -d1 -t0'.format(scriptPath) + '{Enter}', 0.05)\n    time.sleep(3)\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('\u8fd0\u884c\"<Color=Cyan>automation.py -c</Color>\"\u663e\u793a\u9f20\u6807\u5149\u6807\u4e0b\u7684\u63a7\u4ef6\\n\\n')\n    time.sleep(3)\n\n    cmdWindow.SendKeys('\"{}\" -c -t3'.format(scriptPath) + '{Enter}', 0.05)\n    notepadWindow.SetActive()\n    notepadWindow.MoveCursorToMyCenter()\n    time.sleep(3)\n    cmdWindow.SetActive(waitTime = 2)\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('\u8fd0\u884c\"<Color=Cyan>automation.py -a</Color>\"\u663e\u793a\u9f20\u6807\u5149\u6807\u4e0b\u7684\u63a7\u4ef6\u548c\u5b83\u7684\u6240\u6709\u7236\u63a7\u4ef6\\n\\n')\n    time.sleep(3)\n\n    cmdWindow.SendKeys('\"{}\" -a -t3'.format(scriptPath) + '{Enter}', 0.05)\n    notepadWindow.SetActive()\n    notepadWindow.MoveCursorToMyCenter()\n    time.sleep(3)\n    cmdWindow.SetActive(waitTime = 2)\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('\u8fd0\u884c\"<Color=Cyan>automation.py</Color>\"\u663e\u793a\u5f53\u524d\u6fc0\u6d3b\u7a97\u53e3\u548c\u5b83\u7684\u6240\u6709\u5b50\u63a7\u4ef6\\n\\n')\n    time.sleep(3)\n\n    cmdWindow.SendKeys('\"{}\" -t3'.format(scriptPath) + '{Enter}', 0.05)\n    notepadWindow.SetActive()\n    notepadWindow.EditControl().Click()\n    time.sleep(3)\n    cmdWindow.SetActive(waitTime = 2)\n    time.sleep(3)\n\n    thisWindow.SetActive()\n    auto.Logger.WriteLine('\u6f14\u793a\u7ed3\u675f\uff0c\u6309Enter\u9000\u51fa', auto.ConsoleColor.Green)\n    input()", "response": "Demo CN for Chinese language"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef DemoEN():\n    thisWindow = auto.GetConsoleWindow()\n    auto.Logger.ColorfullyWrite('I will run <Color=Cyan>cmd</Color>\\n\\n')\n    time.sleep(3)\n\n    auto.SendKeys('{Win}r')\n    while not isinstance(auto.GetFocusedControl(), auto.EditControl):\n        time.sleep(1)\n    auto.SendKeys('cmd{Enter}')\n    cmdWindow = auto.WindowControl(SubName = 'cmd.exe')\n    rect = cmdWindow.BoundingRectangle\n    auto.DragDrop(rect.left + 50, rect.top + 10, 50, 10)\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('I will run <Color=Cyan>Notepad</Color> and type <Color=Cyan>Hello!!!</Color>\\n\\n')\n    time.sleep(3)\n\n    subprocess.Popen('notepad')\n    notepadWindow = auto.WindowControl(searchDepth = 1, ClassName = 'Notepad')\n    cx, cy = auto.GetScreenSize()\n    notepadWindow.MoveWindow(cx // 2, 20, cx // 2, cy // 2)\n    time.sleep(0.5)\n    notepadWindow.EditControl().SendKeys('Hello!!!', 0.05)\n    time.sleep(1)\n\n    dir = os.path.dirname(__file__)\n    scriptPath = os.path.abspath(os.path.join(dir, '..\\\\automation.py'))\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('run \"<Color=Cyan>automation.py -h</Color>\" to display the help\\n\\n')\n    time.sleep(3)\n\n    cmdWindow.SendKeys('\"{}\" -h'.format(scriptPath) + '{Enter}', 0.05)\n    time.sleep(3)\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('run \"<Color=Cyan>automation.py -r -d1</Color>\" to display the top level windows, desktop\\'s children\\n\\n')\n    time.sleep(3)\n\n    cmdWindow.SendKeys('\"{}\" -r -d1 -t0'.format(scriptPath) + '{Enter}', 0.05)\n    time.sleep(3)\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('run \"<Color=Cyan>automation.py -c</Color>\" to display the control under mouse cursor\\n\\n')\n    time.sleep(3)\n\n    cmdWindow.SendKeys('\"{}\" -c -t3'.format(scriptPath) + '{Enter}', 0.05)\n    notepadWindow.SetActive()\n    notepadWindow.MoveCursorToMyCenter()\n    time.sleep(3)\n    cmdWindow.SetActive(waitTime = 2)\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('run \"<Color=Cyan>automation.py -a</Color>\" to display the control under mouse cursor and its ancestors\\n\\n')\n    time.sleep(3)\n\n    cmdWindow.SendKeys('\"{}\" -a -t3'.format(scriptPath) + '{Enter}', 0.05)\n    notepadWindow.SetActive()\n    notepadWindow.MoveCursorToMyCenter()\n    time.sleep(3)\n    cmdWindow.SetActive(waitTime = 2)\n\n    thisWindow.SetActive()\n    auto.Logger.ColorfullyWrite('run \"<Color=Cyan>automation.py</Color>\" to display the active window\\n\\n')\n    time.sleep(3)\n\n    cmdWindow.SendKeys('\"{}\" -t3'.format(scriptPath) + '{Enter}', 0.05)\n    notepadWindow.SetActive()\n    notepadWindow.EditControl().Click()\n    time.sleep(3)\n    cmdWindow.SetActive(waitTime = 2)\n    time.sleep(3)\n\n    thisWindow.SetActive()\n    auto.Logger.WriteLine('press Enter to exit', auto.ConsoleColor.Green)\n    input()", "response": "This function is used for other language. It is used for all languages."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef threadFunc(root):\n    #print(root)# you cannot use root because it is root control created in main thread\n    th = threading.currentThread()\n    auto.Logger.WriteLine('\\nThis is running in a new thread. {} {}'.format(th.ident, th.name), auto.ConsoleColor.Cyan)\n    time.sleep(2)\n    auto.InitializeUIAutomationInCurrentThread()\n    auto.GetConsoleWindow().CaptureToImage('console_newthread.png')\n    newRoot = auto.GetRootControl()    #ok, root control created in new thread\n    auto.EnumAndLogControl(newRoot, 1)\n    auto.UninitializeUIAutomationInCurrentThread()\n    auto.Logger.WriteLine('\\nThread exits. {} {}'.format(th.ident, th.name), auto.ConsoleColor.Cyan)", "response": "This function is used to create a new thread and use it as a root control."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncompute the saliency map for the specified image and target class.", "response": "def _saliency_map(self, a, image, target, labels, mask, fast=False):\n        \"\"\"Implements Algorithm 3 in manuscript\n\n        \"\"\"\n\n        # pixel influence on target class\n        alphas = a.gradient(image, target) * mask\n\n        # pixel influence on sum of residual classes\n        # (don't evaluate if fast == True)\n        if fast:\n            betas = -np.ones_like(alphas)\n        else:\n            betas = np.sum([\n                a.gradient(image, label) * mask - alphas\n                for label in labels], 0)\n\n        # compute saliency map\n        # (take into account both pos. & neg. perturbations)\n        salmap = np.abs(alphas) * np.abs(betas) * np.sign(alphas * betas)\n\n        # find optimal pixel & direction of perturbation\n        idx = np.argmin(salmap)\n        idx = np.unravel_index(idx, mask.shape)\n        pix_sign = np.sign(alphas)[idx]\n\n        return idx, pix_sign"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalculating the distance between a given image to the original image.", "response": "def normalized_distance(self, image):\n        \"\"\"Calculates the distance of a given image to the\n        original image.\n\n        Parameters\n        ----------\n        image : `numpy.ndarray`\n            The image that should be compared to the original image.\n\n        Returns\n        -------\n        :class:`Distance`\n            The distance between the given image and the original image.\n\n        \"\"\"\n        return self.__distance(\n            self.__original_image_for_distance,\n            image,\n            bounds=self.bounds())"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __is_adversarial(self, image, predictions, in_bounds):\n        is_adversarial = self.__criterion.is_adversarial(\n            predictions, self.__original_class)\n        assert isinstance(is_adversarial, bool) or \\\n            isinstance(is_adversarial, np.bool_)\n        if is_adversarial:\n            is_best, distance = self.__new_adversarial(\n                image, predictions, in_bounds)\n        else:\n            is_best = False\n            distance = None\n        return is_adversarial, is_best, distance", "response": "Interface to criterion. is_adversarial that calls __new_adversarial if necessary."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef channel_axis(self, batch):\n        axis = self.__model.channel_axis()\n        if not batch:\n            axis = axis - 1\n        return axis", "response": "Interface to model. channel_axis for attacks."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn True if _backward and _forward_backward can be called by an attack False otherwise.", "response": "def has_gradient(self):\n        \"\"\"Returns true if _backward and _forward_backward can be called\n        by an attack, False otherwise.\n\n        \"\"\"\n        try:\n            self.__model.gradient\n            self.__model.predictions_and_gradient\n        except AttributeError:\n            return False\n        else:\n            return True"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninterfaces to model. predictions for attacks.", "response": "def predictions(self, image, strict=True, return_details=False):\n        \"\"\"Interface to model.predictions for attacks.\n\n        Parameters\n        ----------\n        image : `numpy.ndarray`\n            Single input with shape as expected by the model\n            (without the batch dimension).\n        strict : bool\n            Controls if the bounds for the pixel values should be checked.\n\n        \"\"\"\n        in_bounds = self.in_bounds(image)\n        assert not strict or in_bounds\n\n        self._total_prediction_calls += 1\n        predictions = self.__model.predictions(image)\n        is_adversarial, is_best, distance = self.__is_adversarial(\n            image, predictions, in_bounds)\n\n        assert predictions.ndim == 1\n        if return_details:\n            return predictions, is_adversarial, is_best, distance\n        else:\n            return predictions, is_adversarial"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninterface to model.batch_predictions for attacks. Parameters ---------- images : `numpy.ndarray` Batch of inputs with shape as expected by the model. greedy : bool Whether the first adversarial should be returned. strict : bool Controls if the bounds for the pixel values should be checked.", "response": "def batch_predictions(\n            self, images, greedy=False, strict=True, return_details=False):\n        \"\"\"Interface to model.batch_predictions for attacks.\n\n        Parameters\n        ----------\n        images : `numpy.ndarray`\n            Batch of inputs with shape as expected by the model.\n        greedy : bool\n            Whether the first adversarial should be returned.\n        strict : bool\n            Controls if the bounds for the pixel values should be checked.\n\n        \"\"\"\n        if strict:\n            in_bounds = self.in_bounds(images)\n            assert in_bounds\n\n        self._total_prediction_calls += len(images)\n        predictions = self.__model.batch_predictions(images)\n\n        assert predictions.ndim == 2\n        assert predictions.shape[0] == images.shape[0]\n\n        if return_details:\n            assert greedy\n\n        adversarials = []\n        for i in range(len(predictions)):\n            if strict:\n                in_bounds_i = True\n            else:\n                in_bounds_i = self.in_bounds(images[i])\n            is_adversarial, is_best, distance = self.__is_adversarial(\n                images[i], predictions[i], in_bounds_i)\n            if is_adversarial and greedy:\n                if return_details:\n                    return predictions, is_adversarial, i, is_best, distance\n                else:\n                    return predictions, is_adversarial, i\n            adversarials.append(is_adversarial)\n\n        if greedy:  # pragma: no cover\n            # no adversarial found\n            if return_details:\n                return predictions, False, None, False, None\n            else:\n                return predictions, False, None\n\n        is_adversarial = np.array(adversarials)\n        assert is_adversarial.ndim == 1\n        assert is_adversarial.shape[0] == images.shape[0]\n\n        return predictions, is_adversarial"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef gradient(self, image=None, label=None, strict=True):\n        assert self.has_gradient()\n\n        if image is None:\n            image = self.__original_image\n        if label is None:\n            label = self.__original_class\n\n        assert not strict or self.in_bounds(image)\n\n        self._total_gradient_calls += 1\n        gradient = self.__model.gradient(image, label)\n\n        assert gradient.shape == image.shape\n        return gradient", "response": "Interface to model. gradient for attacks."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninterfacing to model. predictions_and_gradient for attacks.", "response": "def predictions_and_gradient(\n            self, image=None, label=None, strict=True, return_details=False):\n        \"\"\"Interface to model.predictions_and_gradient for attacks.\n\n        Parameters\n        ----------\n        image : `numpy.ndarray`\n            Single input with shape as expected by the model\n            (without the batch dimension).\n            Defaults to the original image.\n        label : int\n            Label used to calculate the loss that is differentiated.\n            Defaults to the original label.\n        strict : bool\n            Controls if the bounds for the pixel values should be checked.\n\n        \"\"\"\n        assert self.has_gradient()\n\n        if image is None:\n            image = self.__original_image\n        if label is None:\n            label = self.__original_class\n\n        in_bounds = self.in_bounds(image)\n        assert not strict or in_bounds\n\n        self._total_prediction_calls += 1\n        self._total_gradient_calls += 1\n        predictions, gradient = self.__model.predictions_and_gradient(image, label)  # noqa: E501\n        is_adversarial, is_best, distance = self.__is_adversarial(\n            image, predictions, in_bounds)\n\n        assert predictions.ndim == 1\n        assert gradient.shape == image.shape\n        if return_details:\n            return predictions, gradient, is_adversarial, is_best, distance\n        else:\n            return predictions, gradient, is_adversarial"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ninterfaces to model. backward for attacks.", "response": "def backward(self, gradient, image=None, strict=True):\n        \"\"\"Interface to model.backward for attacks.\n\n        Parameters\n        ----------\n        gradient : `numpy.ndarray`\n            Gradient of some loss w.r.t. the logits.\n        image : `numpy.ndarray`\n            Single input with shape as expected by the model\n            (without the batch dimension).\n\n        Returns\n        -------\n        gradient : `numpy.ndarray`\n            The gradient w.r.t the image.\n\n        See Also\n        --------\n        :meth:`gradient`\n\n        \"\"\"\n        assert self.has_gradient()\n        assert gradient.ndim == 1\n\n        if image is None:\n            image = self.__original_image\n\n        assert not strict or self.in_bounds(image)\n\n        self._total_gradient_calls += 1\n        gradient = self.__model.backward(gradient, image)\n\n        assert gradient.shape == image.shape\n        return gradient"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef loss_function(cls, const, a, x, logits, reconstructed_original,\n                      confidence, min_, max_):\n        \"\"\"Returns the loss and the gradient of the loss w.r.t. x,\n        assuming that logits = model(x).\"\"\"\n\n        targeted = a.target_class() is not None\n        if targeted:\n            c_minimize = cls.best_other_class(logits, a.target_class())\n            c_maximize = a.target_class()\n        else:\n            c_minimize = a.original_class\n            c_maximize = cls.best_other_class(logits, a.original_class)\n\n        is_adv_loss = logits[c_minimize] - logits[c_maximize]\n\n        # is_adv is True as soon as the is_adv_loss goes below 0\n        # but sometimes we want additional confidence\n        is_adv_loss += confidence\n        is_adv_loss = max(0, is_adv_loss)\n\n        s = max_ - min_\n        squared_l2_distance = np.sum((x - reconstructed_original)**2) / s**2\n        total_loss = squared_l2_distance + const * is_adv_loss\n\n        # calculate the gradient of total_loss w.r.t. x\n        logits_diff_grad = np.zeros_like(logits)\n        logits_diff_grad[c_minimize] = 1\n        logits_diff_grad[c_maximize] = -1\n        is_adv_loss_grad = a.backward(logits_diff_grad, x)\n        assert is_adv_loss >= 0\n        if is_adv_loss == 0:\n            is_adv_loss_grad = 0\n\n        squared_l2_distance_grad = (2 / s**2) * (x - reconstructed_original)\n\n        total_loss_grad = squared_l2_distance_grad + const * is_adv_loss_grad\n        return total_loss, total_loss_grad", "response": "Returns the loss and gradient of the loss w. r. t. x assuming that logits = model ( x )."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the index of the largest logit ignoring the class that is passed as exclude.", "response": "def best_other_class(logits, exclude):\n        \"\"\"Returns the index of the largest logit, ignoring the class that\n        is passed as `exclude`.\"\"\"\n        other_logits = logits - onehot_like(logits, exclude, value=np.inf)\n        return np.argmax(other_logits)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef name(self):\n        names = (criterion.name() for criterion in self._criteria)\n        return '__'.join(sorted(names))", "response": "Returns the name of the current hierarchy of the entries in alphabetical order."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _difference_map(image, color_axis):\n\n    if color_axis == 2:\n        image = _transpose_image(image)\n    # Derivative in x direction (rows from left to right)\n    dfdx = np.zeros_like(image)\n    # forward difference in first column\n    dfdx[:, :, 0] = image[:, :, 1] - image[:, :, 0]\n    # backwards difference in last column\n    dfdx[:, :, -1] = image[:, :, -1] - image[:, :, -2]\n    # central difference elsewhere\n    dfdx[:, :, 1:-1] = 0.5 * (image[:, :, 2:] - image[:, :, :-2])\n\n    # Derivative in y direction (columns from above to below)\n    dfdy = np.zeros_like(image)\n    # forward difference in first row\n    dfdy[:, 0, :] = image[:, 1, :] - image[:, 0, :]\n    # backwards difference in last row\n    dfdy[:, -1, :] = image[:, -1, :] - image[:, -2, :]\n    # central difference elsewhere\n    dfdy[:, 1:-1, :] = 0.5 * (image[:, 2:, :] - image[:, :-2, :])\n\n    return dfdx, dfdy", "response": "Difference map of the image."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalculating the composition of the function image with the vector field vec_field by interpolation.", "response": "def _compose(image, vec_field, color_axis):\n    \"\"\"Calculate the composition of the function image with the vector\n    field vec_field by interpolation.\n    new_func = compose(image, vec_field)\n    In:\n    image: numpy.ndarray\n        of shape C x h x w with C = 3 or C = 1 (color channels),\n        h, w >= 2, and [type] = 'Float' or 'Double'.\n        Contains the values of a function f: R ^ 2 -> R ^ C\n        on the grid {0, ..., h - 1} x {0, ..., w - 1}.\n    vec_field: numpy.array\n        of shape (h, w, 2)\n    vec_field[y, x, 0] is the x-coordinate of the vector vec_field[y, x]\n    vec_field[y, x, 1] is the y-coordinate of the vector vec_field[y, x]\n    positive x-direction is along rows from left to right\n    positive y-direction is along columns from above to below\n    \"\"\"\n\n    if color_axis == 2:\n        image = _transpose_image(image)\n\n    c, h, w = image.shape  # colors, height, width\n    hrange = np.arange(h)\n    wrange = np.arange(w)\n    MGx, MGy = np.meshgrid(wrange, hrange)\n\n    defMGx = (MGx + vec_field[:, :, 0]).clip(0, w - 1)\n    defMGy = (MGy + vec_field[:, :, 1]).clip(0, h - 1)\n\n    new_image = np.empty_like(image)\n\n    for channel in range(c):\n        # Get a linear interpolation for this color channel.\n        interpolation = RectBivariateSpline(hrange, wrange, image[channel],\n                                            kx=1, ky=1)\n\n        # grid = False since the deformed grid is irregular\n        new_image[channel] = interpolation(defMGy, defMGx, grid=False)\n    if color_axis == 2:\n        return _re_transpose_image(new_image)\n    else:\n        return new_image"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _create_vec_field(fval, gradf, d1x, d2x, color_axis, smooth=0):\n\n    if color_axis == 2:\n        gradf = _transpose_image(gradf)\n\n    c, h, w = gradf.shape  # colors, height, width\n\n    # Sum over color channels\n    alpha1 = np.sum(gradf * d1x, axis=0)\n    alpha2 = np.sum(gradf * d2x, axis=0)\n\n    norm_squared_alpha = (alpha1 ** 2).sum() + (alpha2 ** 2).sum()\n\n    # Smoothing\n    if smooth > 0:\n        alpha1 = gaussian_filter(alpha1, smooth)\n        alpha2 = gaussian_filter(alpha2, smooth)\n        norm_squared_alpha = (alpha1 ** 2).sum() + (alpha2 ** 2).sum()\n        # In theory, we need to apply the filter a second time.\n        alpha1 = gaussian_filter(alpha1, smooth)\n        alpha2 = gaussian_filter(alpha2, smooth)\n\n    vec_field = np.empty((h, w, 2))\n    vec_field[:, :, 0] = -fval * alpha1 / norm_squared_alpha\n    vec_field[:, :, 1] = -fval * alpha2 / norm_squared_alpha\n\n    return vec_field", "response": "Create the deformation vector field for a single resource."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef softmax(logits):\n\n    assert logits.ndim == 1\n\n    # for numerical reasons we subtract the max logit\n    # (mathematically it doesn't matter!)\n    # otherwise exp(logits) might become too large or too small\n    logits = logits - np.max(logits)\n    e = np.exp(logits)\n    return e / np.sum(e)", "response": "Transforms predictions into probability values corresponding to the logits."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef crossentropy(label, logits):\n\n    assert logits.ndim == 1\n\n    # for numerical reasons we subtract the max logit\n    # (mathematically it doesn't matter!)\n    # otherwise exp(logits) might become too large or too small\n    logits = logits - np.max(logits)\n    e = np.exp(logits)\n    s = np.sum(e)\n    ce = np.log(s) - logits[label]\n    return ce", "response": "Calculates the cross - entropy between softmax and onehot."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncalculate the cross - entropy between a batch of logits and a single - hot label.", "response": "def batch_crossentropy(label, logits):\n    \"\"\"Calculates the cross-entropy for a batch of logits.\n\n    Parameters\n    ----------\n    logits : array_like\n        The logits predicted by the model for a batch of inputs.\n    label : int\n        The label describing the target distribution.\n\n    Returns\n    -------\n    np.ndarray\n        The cross-entropy between softmax(logits[i]) and onehot(label)\n        for all i.\n\n    \"\"\"\n\n    assert logits.ndim == 2\n\n    # for numerical reasons we subtract the max logit\n    # (mathematically it doesn't matter!)\n    # otherwise exp(logits) might become too large or too small\n    logits = logits - np.max(logits, axis=1, keepdims=True)\n    e = np.exp(logits)\n    s = np.sum(e, axis=1)\n    ces = np.log(s) - logits[:, label]\n    return ces"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns an example image and its imagenet class label.", "response": "def imagenet_example(shape=(224, 224), data_format='channels_last'):\n    \"\"\" Returns an example image and its imagenet class label.\n\n    Parameters\n    ----------\n    shape : list of integers\n        The shape of the returned image.\n    data_format : str\n        \"channels_first\" or \"channels_last\"\n\n    Returns\n    -------\n    image : array_like\n        The example image.\n\n    label : int\n        The imagenet label associated with the image.\n\n    NOTE: This function is deprecated and will be removed in the future.\n    \"\"\"\n    assert len(shape) == 2\n    assert data_format in ['channels_first', 'channels_last']\n\n    from PIL import Image\n    path = os.path.join(os.path.dirname(__file__), 'example.png')\n    image = Image.open(path)\n    image = image.resize(shape)\n    image = np.asarray(image, dtype=np.float32)\n    image = image[:, :, :3]\n    assert image.shape == shape + (3,)\n    if data_format == 'channels_first':\n        image = np.transpose(image, (2, 0, 1))\n    return image, 282"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a batch of images and labels associated with each example image.", "response": "def samples(dataset='imagenet', index=0, batchsize=1, shape=(224, 224),\n            data_format='channels_last'):\n    ''' Returns a batch of example images and the corresponding labels\n\n    Parameters\n    ----------\n    dataset : string\n        The data set to load (options: imagenet, mnist, cifar10,\n        cifar100, fashionMNIST)\n    index : int\n        For each data set 20 example images exist. The returned batch\n        contains the images with index [index, index + 1, index + 2, ...]\n    batchsize : int\n        Size of batch.\n    shape : list of integers\n        The shape of the returned image (only relevant for Imagenet).\n    data_format : str\n        \"channels_first\" or \"channels_last\"\n\n    Returns\n    -------\n    images : array_like\n        The batch of example images\n\n    labels : array of int\n        The labels associated with the images.\n\n    '''\n    from PIL import Image\n\n    images, labels = [], []\n    basepath = os.path.dirname(__file__)\n    samplepath = os.path.join(basepath, 'data')\n    files = os.listdir(samplepath)\n\n    for idx in range(index, index + batchsize):\n        i = idx % 20\n\n        # get filename and label\n        file = [n for n in files if '{}_{:02d}_'.format(dataset, i) in n][0]\n        label = int(file.split('.')[0].split('_')[-1])\n\n        # open file\n        path = os.path.join(samplepath, file)\n        image = Image.open(path)\n\n        if dataset == 'imagenet':\n            image = image.resize(shape)\n\n        image = np.asarray(image, dtype=np.float32)\n\n        if dataset != 'mnist' and data_format == 'channels_first':\n            image = np.transpose(image, (2, 0, 1))\n\n        images.append(image)\n        labels.append(label)\n\n    labels = np.array(labels)\n    images = np.stack(images)\n    return images, labels"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef onehot_like(a, index, value=1):\n\n    x = np.zeros_like(a)\n    x[index] = value\n    return x", "response": "Creates an array like a with all values set to 0 except one."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the output image for a given image.", "response": "def _get_output(self, a, image):\n        \"\"\" Looks up the precomputed adversarial image for a given image.\n\n        \"\"\"\n        sd = np.square(self._input_images - image)\n        mses = np.mean(sd, axis=tuple(range(1, sd.ndim)))\n        index = np.argmin(mses)\n\n        # if we run into numerical problems with this approach, we might\n        # need to add a very tiny threshold here\n        if mses[index] > 0:\n            raise ValueError('No precomputed output image for this image')\n        return self._output_images[index]"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef gradient(self, image, label):\n        _, gradient = self.predictions_and_gradient(image, label)\n        return gradient", "response": "Calculates the gradient of the cross - entropy loss w. r. t. the image."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nclones a remote git repository to a local path.", "response": "def clone(git_uri):\n    \"\"\"\n    Clone a remote git repository to a local path.\n\n    :param git_uri: the URI to the git repository to be cloned\n    :return: the generated local path where the repository has been cloned to\n    \"\"\"\n    hash_digest = sha256_hash(git_uri)\n    local_path = home_directory_path(FOLDER, hash_digest)\n    exists_locally = path_exists(local_path)\n\n    if not exists_locally:\n        _clone_repo(git_uri, local_path)\n    else:\n        logging.info(  # pragma: no cover\n            \"Git repository already exists locally.\")  # pragma: no cover\n\n    return local_path"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\npolling the socket and return a list of resources that can be read from the pipe.", "response": "def poll(self, timeout=0.0):\n    \"\"\"Modified version of poll() from asyncore module\"\"\"\n    if self.sock_map is None:\n      Log.warning(\"Socket map is not registered to Gateway Looper\")\n    readable_lst = []\n    writable_lst = []\n    error_lst = []\n\n    if self.sock_map is not None:\n      for fd, obj in self.sock_map.items():\n        is_r = obj.readable()\n        is_w = obj.writable()\n        if is_r:\n          readable_lst.append(fd)\n        if is_w and not obj.accepting:\n          writable_lst.append(fd)\n        if is_r or is_w:\n          error_lst.append(fd)\n\n    # Add wakeup fd\n    readable_lst.append(self.pipe_r)\n\n    Log.debug(\"Will select() with timeout: \" + str(timeout) + \", with map: \" + str(self.sock_map))\n    try:\n      readable_lst, writable_lst, error_lst = \\\n        select.select(readable_lst, writable_lst, error_lst, timeout)\n    except select.error as err:\n      Log.debug(\"Trivial error: \" + str(err))\n      if err.args[0] != errno.EINTR:\n        raise\n      else:\n        return\n    Log.debug(\"Selected [r]: \" + str(readable_lst) +\n              \" [w]: \" + str(writable_lst) + \" [e]: \" + str(error_lst))\n\n    if self.pipe_r in readable_lst:\n      Log.debug(\"Read from pipe\")\n      os.read(self.pipe_r, 1024)\n      readable_lst.remove(self.pipe_r)\n\n    if self.sock_map is not None:\n      for fd in readable_lst:\n        obj = self.sock_map.get(fd)\n        if obj is None:\n          continue\n        asyncore.read(obj)\n\n      for fd in writable_lst:\n        obj = self.sock_map.get(fd)\n        if obj is None:\n          continue\n        asyncore.write(obj)\n\n      for fd in error_lst:\n        obj = self.sock_map.get(fd)\n        if obj is None:\n          continue\n        # pylint: disable=W0212\n        asyncore._exception(obj)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwriting a success response to the response stream.", "response": "def write_success_response(self, result):\n    \"\"\"\n    Result may be a python dictionary, array or a primitive type\n    that can be converted to JSON for writing back the result.\n    \"\"\"\n    response = self.make_success_response(result)\n    now = time.time()\n    spent = now - self.basehandler_starttime\n    response[constants.RESPONSE_KEY_EXECUTION_TIME] = spent\n    self.write_json_response(response)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrite the message as part of the response and sets the status to 404.", "response": "def write_error_response(self, message):\n    \"\"\"\n    Writes the message as part of the response and sets 404 status.\n    \"\"\"\n    self.set_status(404)\n    response = self.make_error_response(str(message))\n    now = time.time()\n    spent = now - self.basehandler_starttime\n    response[constants.RESPONSE_KEY_EXECUTION_TIME] = spent\n    self.write_json_response(response)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef write_json_response(self, response):\n    self.write(tornado.escape.json_encode(response))\n    self.set_header(\"Content-Type\", \"application/json\")", "response": "write back json response"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef make_response(self, status):\n    response = {\n        constants.RESPONSE_KEY_STATUS: status,\n        constants.RESPONSE_KEY_VERSION: constants.API_VERSION,\n        constants.RESPONSE_KEY_EXECUTION_TIME: 0,\n        constants.RESPONSE_KEY_MESSAGE: \"\",\n    }\n    return response", "response": "Makes the base dict for the response."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef make_success_response(self, result):\n    response = self.make_response(constants.RESPONSE_STATUS_SUCCESS)\n    response[constants.RESPONSE_KEY_RESULT] = result\n    return response", "response": "Makes a python dict corresponding to the successful\n    response."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nmake a response dict that contains a status of failure.", "response": "def make_error_response(self, message):\n    \"\"\"\n    Makes the python dict corresponding to the\n    JSON that needs to be sent for a failed\n    response. Message is the message that is\n    sent as the reason for failure.\n    \"\"\"\n    response = self.make_response(constants.RESPONSE_STATUS_FAILURE)\n    response[constants.RESPONSE_KEY_MESSAGE] = message\n    return response"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_argument_cluster(self):\n    try:\n      return self.get_argument(constants.PARAM_CLUSTER)\n    except tornado.web.MissingArgumentError as e:\n      raise Exception(e.log_message)", "response": "Helper function to get request argument cluster."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_argument_topology(self):\n    try:\n      topology = self.get_argument(constants.PARAM_TOPOLOGY)\n      return topology\n    except tornado.web.MissingArgumentError as e:\n      raise Exception(e.log_message)", "response": "Helper function to get the topology argument."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_argument_component(self):\n    try:\n      component = self.get_argument(constants.PARAM_COMPONENT)\n      return component\n    except tornado.web.MissingArgumentError as e:\n      raise Exception(e.log_message)", "response": "Helper function to get component argument."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_argument_instance(self):\n    try:\n      instance = self.get_argument(constants.PARAM_INSTANCE)\n      return instance\n    except tornado.web.MissingArgumentError as e:\n      raise Exception(e.log_message)", "response": "Helper function to get instance argument."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_argument_starttime(self):\n    try:\n      starttime = self.get_argument(constants.PARAM_STARTTIME)\n      return starttime\n    except tornado.web.MissingArgumentError as e:\n      raise Exception(e.log_message)", "response": "Helper function to get starttime argument."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_argument_endtime(self):\n    try:\n      endtime = self.get_argument(constants.PARAM_ENDTIME)\n      return endtime\n    except tornado.web.MissingArgumentError as e:\n      raise Exception(e.log_message)", "response": "Helper function to get endtime argument."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_argument_offset(self):\n    try:\n      offset = self.get_argument(constants.PARAM_OFFSET)\n      return offset\n    except tornado.web.MissingArgumentError as e:\n      raise Exception(e.log_message)", "response": "Helper function to get offset argument."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef validateInterval(self, startTime, endTime):\n    start = int(startTime)\n    end = int(endTime)\n    if start > end:\n      raise Exception(\"starttime is greater than endtime.\")", "response": "Validate that the start and end times are in the interval."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nstart the connection to the Heron server.", "response": "def start_connect(self):\n    \"\"\"Tries to connect to the Heron Server\n\n    ``loop()`` method needs to be called after this.\n    \"\"\"\n    Log.debug(\"In start_connect() of %s\" % self._get_classname())\n    # TODO: specify buffer size, exception handling\n    self.create_socket(socket.AF_INET, socket.SOCK_STREAM)\n\n    # when ready, handle_connect is called\n    self._connecting = True\n    self.connect(self.endpoint)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nregisters a protobuf message builder that this client wants to receive", "response": "def register_on_message(self, msg_builder):\n    \"\"\"Registers protobuf message builders that this client wants to receive\n\n    :param msg_builder: callable to create a protobuf message that this client wants to receive\n    \"\"\"\n    message = msg_builder()\n    Log.debug(\"In register_on_message(): %s\" % message.DESCRIPTOR.full_name)\n    self.registered_message_map[message.DESCRIPTOR.full_name] = msg_builder"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsending a request to the server.", "response": "def send_request(self, request, context, response_type, timeout_sec):\n    \"\"\"Sends a request message (REQID is non-zero)\"\"\"\n    # generates a unique request id\n    reqid = REQID.generate()\n    Log.debug(\"%s: In send_request() with REQID: %s\" % (self._get_classname(), str(reqid)))\n    # register response message type\n    self.response_message_map[reqid] = response_type\n    self.context_map[reqid] = context\n\n    # Add timeout for this request if necessary\n    if timeout_sec > 0:\n      def timeout_task():\n        self.handle_timeout(reqid)\n      self.looper.register_timer_task_in_sec(timeout_task, timeout_sec)\n\n    outgoing_pkt = OutgoingPacket.create_packet(reqid, request)\n    self._send_packet(outgoing_pkt)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsending a message to the server.", "response": "def send_message(self, message):\n    \"\"\"Sends a message (REQID is zero)\"\"\"\n    Log.debug(\"In send_message() of %s\" % self._get_classname())\n    outgoing_pkt = OutgoingPacket.create_packet(REQID.generate_zero(), message)\n    self._send_packet(outgoing_pkt)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef handle_timeout(self, reqid):\n    if reqid in self.context_map:\n      context = self.context_map.pop(reqid)\n      self.response_message_map.pop(reqid)\n      self.on_response(StatusCode.TIMEOUT_ERROR, context, None)", "response": "Handles a timeout error."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_tar(tar_filename, files, config_dir, config_files):\n  '''\n  Create a tar file with a given set of files\n  '''\n  with contextlib.closing(tarfile.open(tar_filename, 'w:gz', dereference=True)) as tar:\n    for filename in files:\n      if os.path.isfile(filename):\n        tar.add(filename, arcname=os.path.basename(filename))\n      else:\n        raise Exception(\"%s is not an existing file\" % filename)\n\n    if os.path.isdir(config_dir):\n      tar.add(config_dir, arcname=get_heron_sandbox_conf_dir())\n    else:\n      raise Exception(\"%s is not an existing directory\" % config_dir)\n\n    for filename in config_files:\n      if os.path.isfile(filename):\n        arcfile = os.path.join(get_heron_sandbox_conf_dir(), os.path.basename(filename))\n        tar.add(filename, arcname=arcfile)\n      else:\n        raise Exception(\"%s is not an existing file\" % filename)", "response": "Create a tar file with a given set of files and a given set of config files"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nretrieving the given subparser from parser Returns the given subparser from parser", "response": "def get_subparser(parser, command):\n  '''\n  Retrieve the given subparser from parser\n  '''\n  # pylint: disable=protected-access\n  subparsers_actions = [action for action in parser._actions\n                        if isinstance(action, argparse._SubParsersAction)]\n\n  # there will probably only be one subparser_action,\n  # but better save than sorry\n  for subparsers_action in subparsers_actions:\n    # get all subparsers\n    for choice, subparser in subparsers_action.choices.items():\n      if choice == command:\n        return subparser\n  return None"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets all the heron lib jars with the absolute paths", "response": "def get_heron_libs(local_jars):\n  \"\"\"Get all the heron lib jars with the absolute paths\"\"\"\n  heron_lib_dir = get_heron_lib_dir()\n  heron_libs = [os.path.join(heron_lib_dir, f) for f in local_jars]\n  return heron_libs"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nparse cluster role and environment variables.", "response": "def parse_cluster_role_env(cluster_role_env, config_path):\n  \"\"\"Parse cluster/[role]/[environ], supply default, if not provided, not required\"\"\"\n  parts = cluster_role_env.split('/')[:3]\n  if not os.path.isdir(config_path):\n    Log.error(\"Config path cluster directory does not exist: %s\" % config_path)\n    raise Exception(\"Invalid config path\")\n\n  # if cluster/role/env is not completely provided, check further\n  if len(parts) < 3:\n\n    cli_conf_file = os.path.join(config_path, CLIENT_YAML)\n\n    # if client conf doesn't exist, use default value\n    if not os.path.isfile(cli_conf_file):\n      if len(parts) == 1:\n        parts.append(getpass.getuser())\n      if len(parts) == 2:\n        parts.append(ENVIRON)\n    else:\n      cli_confs = {}\n      with open(cli_conf_file, 'r') as conf_file:\n        tmp_confs = yaml.load(conf_file)\n        # the return value of yaml.load can be None if conf_file is an empty file\n        if tmp_confs is not None:\n          cli_confs = tmp_confs\n        else:\n          print(\"Failed to read: %s due to it is empty\" % (CLIENT_YAML))\n\n      # if role is required but not provided, raise exception\n      if len(parts) == 1:\n        if (ROLE_REQUIRED in cli_confs) and (cli_confs[ROLE_REQUIRED] is True):\n          raise Exception(\"role required but not provided (cluster/role/env = %s). See %s in %s\"\n                          % (cluster_role_env, ROLE_REQUIRED, cli_conf_file))\n        else:\n          parts.append(getpass.getuser())\n\n      # if environ is required but not provided, raise exception\n      if len(parts) == 2:\n        if (ENV_REQUIRED in cli_confs) and (cli_confs[ENV_REQUIRED] is True):\n          raise Exception(\"environ required but not provided (cluster/role/env = %s). See %s in %s\"\n                          % (cluster_role_env, ENV_REQUIRED, cli_conf_file))\n        else:\n          parts.append(ENVIRON)\n\n  # if cluster or role or environ is empty, print\n  if len(parts[0]) == 0 or len(parts[1]) == 0 or len(parts[2]) == 0:\n    print(\"Failed to parse\")\n    sys.exit(1)\n\n  return (parts[0], parts[1], parts[2])"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nparse cluster role env string supply empty string if not provided", "response": "def get_cluster_role_env(cluster_role_env):\n  \"\"\"Parse cluster/[role]/[environ], supply empty string, if not provided\"\"\"\n  parts = cluster_role_env.split('/')[:3]\n  if len(parts) == 3:\n    return (parts[0], parts[1], parts[2])\n\n  if len(parts) == 2:\n    return (parts[0], parts[1], \"\")\n\n  if len(parts) == 1:\n    return (parts[0], \"\", \"\")\n\n  return (\"\", \"\", \"\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef direct_mode_cluster_role_env(cluster_role_env, config_path):\n\n  # otherwise, get the client.yaml file\n  cli_conf_file = os.path.join(config_path, CLIENT_YAML)\n\n  # if client conf doesn't exist, use default value\n  if not os.path.isfile(cli_conf_file):\n    return True\n\n  client_confs = {}\n  with open(cli_conf_file, 'r') as conf_file:\n    client_confs = yaml.load(conf_file)\n\n    # the return value of yaml.load can be None if conf_file is an empty file\n    if not client_confs:\n      return True\n\n    # if role is required but not provided, raise exception\n    role_present = True if len(cluster_role_env[1]) > 0 else False\n    if ROLE_REQUIRED in client_confs and client_confs[ROLE_REQUIRED] and not role_present:\n      raise Exception(\"role required but not provided (cluster/role/env = %s). See %s in %s\"\n                      % (cluster_role_env, ROLE_REQUIRED, cli_conf_file))\n\n    # if environ is required but not provided, raise exception\n    environ_present = True if len(cluster_role_env[2]) > 0 else False\n    if ENV_REQUIRED in client_confs and client_confs[ENV_REQUIRED] and not environ_present:\n      raise Exception(\"environ required but not provided (cluster/role/env = %s). See %s in %s\"\n                      % (cluster_role_env, ENV_REQUIRED, cli_conf_file))\n\n  return True", "response": "Check cluster role and environment"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchecks cluster role and environment are present in config_map.", "response": "def server_mode_cluster_role_env(cluster_role_env, config_map):\n  \"\"\"Check cluster/[role]/[environ], if they are required\"\"\"\n\n  cmap = config_map[cluster_role_env[0]]\n\n  # if role is required but not provided, raise exception\n  role_present = True if len(cluster_role_env[1]) > 0 else False\n  if ROLE_KEY in cmap and cmap[ROLE_KEY] and not role_present:\n    raise Exception(\"role required but not provided (cluster/role/env = %s).\"\\\n        % (cluster_role_env))\n\n  # if environ is required but not provided, raise exception\n  environ_present = True if len(cluster_role_env[2]) > 0 else False\n  if ENVIRON_KEY in cmap and cmap[ENVIRON_KEY] and not environ_present:\n    raise Exception(\"environ required but not provided (cluster/role/env = %s).\"\\\n        % (cluster_role_env))\n\n  return True"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef defaults_cluster_role_env(cluster_role_env):\n  if len(cluster_role_env[1]) == 0 and len(cluster_role_env[2]) == 0:\n    return (cluster_role_env[0], getpass.getuser(), ENVIRON)\n\n  return (cluster_role_env[0], cluster_role_env[1], cluster_role_env[2])", "response": "Default role environment for cluster"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_override_config_and_write_file(namespace):\n  overrides = parse_override_config(namespace)\n  try:\n    tmp_dir = tempfile.mkdtemp()\n    override_config_file = os.path.join(tmp_dir, OVERRIDE_YAML)\n    with open(override_config_file, 'w') as f:\n      f.write(yaml.dump(overrides))\n\n    return override_config_file\n  except Exception as e:\n    raise Exception(\"Failed to parse override config: %s\" % str(e))", "response": "Parse the command line for overriding the defaults and\n create an override file."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing the command line for overriding the defaults", "response": "def parse_override_config(namespace):\n  \"\"\"Parse the command line for overriding the defaults\"\"\"\n  overrides = dict()\n  for config in namespace:\n    kv = config.split(\"=\")\n    if len(kv) != 2:\n      raise Exception(\"Invalid config property format (%s) expected key=value\" % config)\n    if kv[1] in ['true', 'True', 'TRUE']:\n      overrides[kv[0]] = True\n    elif kv[1] in ['false', 'False', 'FALSE']:\n      overrides[kv[0]] = False\n    else:\n      overrides[kv[0]] = kv[1]\n  return overrides"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_java_path():\n  java_home = os.environ.get(\"JAVA_HOME\")\n  return os.path.join(java_home, BIN_DIR, \"java\")", "response": "Get the path of java executable"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking if the java home set is set", "response": "def check_java_home_set():\n  \"\"\"Check if the java home set\"\"\"\n  # check if environ variable is set\n  if \"JAVA_HOME\" not in os.environ:\n    Log.error(\"JAVA_HOME not set\")\n    return False\n\n  # check if the value set is correct\n  java_path = get_java_path()\n  if os.path.isfile(java_path) and os.access(java_path, os.X_OK):\n    return True\n\n  Log.error(\"JAVA_HOME/bin/java either does not exist or not an executable\")\n  return False"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef check_release_file_exists():\n  release_file = get_heron_release_file()\n\n  # if the file does not exist and is not a file\n  if not os.path.isfile(release_file):\n    Log.error(\"Required file not found: %s\" % release_file)\n    return False\n\n  return True", "response": "Check if the release. yaml file exists"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef print_build_info(zipped_pex=False):\n  if zipped_pex:\n    release_file = get_zipped_heron_release_file()\n  else:\n    release_file = get_heron_release_file()\n\n  with open(release_file) as release_info:\n    release_map = yaml.load(release_info)\n    release_items = sorted(release_map.items(), key=lambda tup: tup[0])\n    for key, value in release_items:\n      print(\"%s : %s\" % (key, value))", "response": "Print build_info from release. yaml\n ethernet_file."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_version_number(zipped_pex=False):\n  if zipped_pex:\n    release_file = get_zipped_heron_release_file()\n  else:\n    release_file = get_heron_release_file()\n  with open(release_file) as release_info:\n    for line in release_info:\n      trunks = line[:-1].split(' ')\n      if trunks[0] == 'heron.build.version':\n        return trunks[-1].replace(\"'\", \"\")\n    return 'unknown'", "response": "Print version from release. yaml\n Return version number from release. yaml\n"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef insert_bool(param, command_args):\n  '''\n  :param param:\n  :param command_args:\n  :return:\n  '''\n  index = 0\n  found = False\n  for lelem in command_args:\n    if lelem == '--' and not found:\n      break\n    if lelem == param:\n      found = True\n      break\n    index = index + 1\n\n  if found:\n    command_args.insert(index + 1, 'True')\n  return command_args", "response": "Insert boolean into command_args."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get(self):\n    try:\n      cluster = self.get_argument_cluster()\n      environ = self.get_argument_environ()\n      role = self.get_argument_role()\n      topology_name = self.get_argument_topology()\n      component = self.get_argument_component()\n      topology = self.tracker.getTopologyByClusterRoleEnvironAndName(\n          cluster, role, environ, topology_name)\n      instances = self.get_arguments(constants.PARAM_INSTANCE)\n      exceptions_summary = yield tornado.gen.Task(self.getComponentExceptionSummary,\n                                                  topology.tmaster, component, instances)\n      self.write_success_response(exceptions_summary)\n    except Exception as e:\n      Log.debug(traceback.format_exc())\n      self.write_error_response(e)", "response": "Get the next task in the chain"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the summary of exceptions for a given component.", "response": "def getComponentExceptionSummary(self, tmaster, component_name, instances=[], callback=None):\n    \"\"\"\n    Get the summary of exceptions for component_name and list of instances.\n    Empty instance list will fetch all exceptions.\n    \"\"\"\n    if not tmaster or not tmaster.host or not tmaster.stats_port:\n      return\n    exception_request = tmaster_pb2.ExceptionLogRequest()\n    exception_request.component_name = component_name\n    if len(instances) > 0:\n      exception_request.instances.extend(instances)\n    request_str = exception_request.SerializeToString()\n    port = str(tmaster.stats_port)\n    host = tmaster.host\n    url = \"http://{0}:{1}/exceptionsummary\".format(host, port)\n    Log.debug(\"Creating request object.\")\n    request = tornado.httpclient.HTTPRequest(url,\n                                             body=request_str,\n                                             method='POST',\n                                             request_timeout=5)\n    Log.debug('Making HTTP call to fetch exceptionsummary url: %s', url)\n    try:\n      client = tornado.httpclient.AsyncHTTPClient()\n      result = yield client.fetch(request)\n      Log.debug(\"HTTP call complete.\")\n    except tornado.httpclient.HTTPError as e:\n      raise Exception(str(e))\n\n    # Check the response code - error if it is in 400s or 500s\n    responseCode = result.code\n    if responseCode >= 400:\n      message = \"Error in getting exceptions from Tmaster, code: \" + responseCode\n      Log.error(message)\n      raise tornado.gen.Return({\n          \"message\": message\n      })\n\n    # Parse the response from tmaster.\n    exception_response = tmaster_pb2.ExceptionLogResponse()\n    exception_response.ParseFromString(result.body)\n\n    if exception_response.status.status == common_pb2.NOTOK:\n      if exception_response.status.HasField(\"message\"):\n        raise tornado.gen.Return({\n            \"message\": exception_response.status.message\n        })\n\n    # Send response\n    ret = []\n    for exception_log in exception_response.exceptions:\n      ret.append({'class_name': exception_log.stacktrace,\n                  'lasttime': exception_log.lasttime,\n                  'firsttime': exception_log.firsttime,\n                  'count': str(exception_log.count)})\n    raise tornado.gen.Return(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get(self, cluster, environ, topology):\n    '''\n    :param cluster:\n    :param environ:\n    :param topology:\n    :return:\n    '''\n    # pylint: disable=no-member\n    options = dict(\n        cluster=cluster,\n        environ=environ,\n        topology=topology,\n        active=\"topologies\",\n        function=common.className,\n        baseUrl=self.baseUrl)\n    self.render(\"config.html\", **options)", "response": "Get the current configuration."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get(self):\n    '''\n    :return:\n    '''\n    clusters = yield access.get_clusters()\n\n    # pylint: disable=no-member\n    options = dict(\n        topologies=[],  # no topologies\n        clusters=[str(cluster) for cluster in clusters],\n        active=\"topologies\",  # active icon the nav bar\n        function=common.className,\n        baseUrl=self.baseUrl\n    )\n\n    # send the all topologies page\n    self.render(\"topologies.html\", **options)", "response": "Get all topologies from the current page"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting a single topology page.", "response": "def get(self, cluster, environ, topology):\n    '''\n    :param cluster:\n    :param environ:\n    :param topology:\n    :return:\n    '''\n\n    # fetch the execution of the topology asynchronously\n    execution_state = yield access.get_execution_state(cluster, environ, topology)\n\n    # fetch scheduler location of the topology\n    scheduler_location = yield access.get_scheduler_location(cluster, environ, topology)\n\n    job_page_link = scheduler_location[\"job_page_link\"]\n\n    # convert the topology launch time to display format\n    launched_at = datetime.utcfromtimestamp(execution_state['submission_time'])\n    launched_time = launched_at.strftime('%Y-%m-%d %H:%M:%S UTC')\n\n    # pylint: disable=no-member\n    options = dict(\n        cluster=cluster,\n        environ=environ,\n        topology=topology,\n        execution_state=execution_state,\n        launched=launched_time,\n        status=\"running\" if random.randint(0, 1) else \"errors\",\n        active=\"topologies\",\n        job_page_link=job_page_link,\n        function=common.className,\n        baseUrl=self.baseUrl\n    )\n\n    # send the single topology page\n    self.render(\"topology.html\", **options)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget a container file.", "response": "def get(self, cluster, environ, topology, container):\n    '''\n    :param cluster:\n    :param environ:\n    :param topology:\n    :param container:\n    :return:\n    '''\n    offset = self.get_argument(\"offset\")\n    length = self.get_argument(\"length\")\n    path = self.get_argument(\"path\")\n\n    data = yield access.get_container_file_data(cluster, environ, topology, container, path,\n                                                offset, length)\n\n    self.write(data)\n    self.finish()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting a single file from the server.", "response": "def get(self, cluster, environ, topology, container):\n    '''\n    :param cluster:\n    :param environ:\n    :param topology:\n    :param container:\n    :return:\n    '''\n    path = self.get_argument(\"path\", default=\".\")\n    data = yield access.get_filestats(cluster, environ, topology, container, path)\n\n    options = dict(\n        cluster=cluster,\n        environ=environ,\n        topology=topology,\n        container=container,\n        path=path,\n        filestats=data,\n        baseUrl=self.baseUrl)\n    self.render(\"browse.html\", **options)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget a file from the container.", "response": "def get(self, cluster, environ, topology, container):\n    '''\n    :param cluster:\n    :param environ:\n    :param topology:\n    :param container:\n    :return:\n    '''\n    # If the file is large, we want to abandon downloading\n    # if user cancels the requests.\n    # pylint: disable=attribute-defined-outside-init\n    self.connection_closed = False\n\n    path = self.get_argument(\"path\")\n    filename = path.split(\"/\")[-1]\n    self.set_header(\"Content-Disposition\", \"attachment; filename=%s\" % filename)\n\n    # Download the files in chunks. We are downloading from Tracker,\n    # which in turns downloads from heron-shell. This much indirection\n    # means that if we use static file downloading, the whole files would\n    # be cached in memory before it can be sent downstream. Hence, we reuse\n    # the file data API to read in chunks until the EOF, or until the download\n    # is cancelled by user.\n\n    # 4 MB gives good enough chunk size giving good speed for small files.\n    # If files are large, a single threaded download may not be enough.\n    file_download_url = access.get_container_file_download_url(cluster, environ,\n                                                               topology, container, path)\n\n    Log.debug(\"file download url: %s\", str(file_download_url))\n    def streaming_callback(chunk):\n      self.write(chunk)\n      self.flush()\n\n    http_client = tornado.httpclient.AsyncHTTPClient()\n    yield http_client.fetch(file_download_url, streaming_callback=streaming_callback)\n    self.finish()"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nregistering a watch with the topology.", "response": "def register_watch(self, callback):\n    \"\"\"\n    Returns the UUID with which the watch is\n    registered. This UUID can be used to unregister\n    the watch.\n    Returns None if watch could not be registered.\n\n    The argument 'callback' must be a function that takes\n    exactly one argument, the topology on which\n    the watch was triggered.\n    Note that the watch will be unregistered in case\n    it raises any Exception the first time.\n\n    This callback is also called at the time\n    of registration.\n    \"\"\"\n    RETRY_COUNT = 5\n    # Retry in case UID is previously\n    # generated, just in case...\n    for _ in range(RETRY_COUNT):\n      # Generate a random UUID.\n      uid = uuid.uuid4()\n      if uid not in self.watches:\n        Log.info(\"Registering a watch with uid: \" + str(uid))\n        try:\n          callback(self)\n        except Exception as e:\n          Log.error(\"Caught exception while triggering callback: \" + str(e))\n          Log.debug(traceback.format_exc())\n          return None\n        self.watches[uid] = callback\n        return uid\n    return None"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef unregister_watch(self, uid):\n    # Do not raise an error if UUID is\n    # not present in the watches.\n    Log.info(\"Unregister a watch with uid: \" + str(uid))\n    self.watches.pop(uid, None)", "response": "Unregister a watch with the given UUID."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalling all the callbacks. call all the callbacks. unregister the corresponding watch.", "response": "def trigger_watches(self):\n    \"\"\"\n    Call all the callbacks.\n    If any callback raises an Exception,\n    unregister the corresponding watch.\n    \"\"\"\n    to_remove = []\n    for uid, callback in self.watches.items():\n      try:\n        callback(self)\n      except Exception as e:\n        Log.error(\"Caught exception while triggering callback: \" + str(e))\n        Log.debug(traceback.format_exc())\n        to_remove.append(uid)\n\n    for uid in to_remove:\n      self.unregister_watch(uid)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef set_physical_plan(self, physical_plan):\n    if not physical_plan:\n      self.physical_plan = None\n      self.id = None\n    else:\n      self.physical_plan = physical_plan\n      self.id = physical_plan.topology.id\n    self.trigger_watches()", "response": "set the physical plan of the object"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset the execution state of the cluster", "response": "def set_execution_state(self, execution_state):\n    \"\"\" set exectuion state \"\"\"\n    if not execution_state:\n      self.execution_state = None\n      self.cluster = None\n      self.environ = None\n    else:\n      self.execution_state = execution_state\n      cluster, environ = self.get_execution_state_dc_environ(execution_state)\n      self.cluster = cluster\n      self.environ = environ\n      self.zone = cluster\n    self.trigger_watches()"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef num_instances(self):\n    num = 0\n\n    # Get all the components\n    components = self.spouts() + self.bolts()\n\n    # Get instances for each worker\n    for component in components:\n      config = component.comp.config\n      for kvs in config.kvs:\n        if kvs.key == api_constants.TOPOLOGY_COMPONENT_PARALLELISM:\n          num += int(kvs.value)\n          break\n\n    return num", "response": "Returns the number of instances for the current project."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a list of all the machines that this topology is running on.", "response": "def get_machines(self):\n    \"\"\"\n    Get all the machines that this topology is running on.\n    These are the hosts of all the stmgrs.\n    \"\"\"\n    if self.physical_plan:\n      stmgrs = list(self.physical_plan.stmgrs)\n      return map(lambda s: s.host_name, stmgrs)\n    return []"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_status(self):\n    status = None\n    if self.physical_plan and self.physical_plan.topology:\n      status = self.physical_plan.topology.state\n\n    if status == 1:\n      return \"Running\"\n    elif status == 2:\n      return \"Paused\"\n    elif status == 3:\n      return \"Killed\"\n    else:\n      return \"Unknown\"", "response": "Returns the status of the topology."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef convert_pb_kvs(kvs, include_non_primitives=True):\n  config = {}\n  for kv in kvs:\n    if kv.value:\n      config[kv.key] = kv.value\n    elif kv.serialized_value:\n      # add serialized_value support for python values (fixme)\n\n      # is this a serialized java object\n      if topology_pb2.JAVA_SERIALIZED_VALUE == kv.type:\n        jv = _convert_java_value(kv, include_non_primitives=include_non_primitives)\n        if jv is not None:\n          config[kv.key] = jv\n      else:\n        config[kv.key] = _raw_value(kv)\n  return config", "response": "Converts protobuf kvs to dict\n"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef synch_topologies(self):\n    self.state_managers = statemanagerfactory.get_all_state_managers(self.config.statemgr_config)\n    try:\n      for state_manager in self.state_managers:\n        state_manager.start()\n    except Exception as ex:\n      Log.error(\"Found exception while initializing state managers: %s. Bailing out...\" % ex)\n      traceback.print_exc()\n      sys.exit(1)\n\n    # pylint: disable=deprecated-lambda\n    def on_topologies_watch(state_manager, topologies):\n      \"\"\"watch topologies\"\"\"\n      Log.info(\"State watch triggered for topologies.\")\n      Log.debug(\"Topologies: \" + str(topologies))\n      existingTopologies = self.getTopologiesForStateLocation(state_manager.name)\n      existingTopNames = map(lambda t: t.name, existingTopologies)\n      Log.debug(\"Existing topologies: \" + str(existingTopNames))\n      for name in existingTopNames:\n        if name not in topologies:\n          Log.info(\"Removing topology: %s in rootpath: %s\",\n                   name, state_manager.rootpath)\n          self.removeTopology(name, state_manager.name)\n\n      for name in topologies:\n        if name not in existingTopNames:\n          self.addNewTopology(state_manager, name)\n\n    for state_manager in self.state_managers:\n      # The callback function with the bound\n      # state_manager as first variable.\n      onTopologiesWatch = partial(on_topologies_watch, state_manager)\n      state_manager.get_topologies(onTopologiesWatch)", "response": "Synchronize the topologies with the state manager s state managers."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getTopologyByClusterRoleEnvironAndName(self, cluster, role, environ, topologyName):\n    topologies = list(filter(lambda t: t.name == topologyName\n                             and t.cluster == cluster\n                             and (not role or t.execution_state.role == role)\n                             and t.environ == environ, self.topologies))\n    if not topologies or len(topologies) > 1:\n      if role is not None:\n        raise Exception(\"Topology not found for {0}, {1}, {2}, {3}\".format(\n            cluster, role, environ, topologyName))\n      else:\n        raise Exception(\"Topology not found for {0}, {1}, {2}\".format(\n            cluster, environ, topologyName))\n\n    # There is only one topology which is returned.\n    return topologies[0]", "response": "Find and return the topology given its cluster environ topology name and topology name and optional role."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef getTopologiesForStateLocation(self, name):\n    return filter(lambda t: t.state_manager_name == name, self.topologies)", "response": "Returns all the topologies for a given state location."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef addNewTopology(self, state_manager, topologyName):\n    topology = Topology(topologyName, state_manager.name)\n    Log.info(\"Adding new topology: %s, state_manager: %s\",\n             topologyName, state_manager.name)\n    self.topologies.append(topology)\n\n    # Register a watch on topology and change\n    # the topologyInfo on any new change.\n    topology.register_watch(self.setTopologyInfo)\n\n    def on_topology_pplan(data):\n      \"\"\"watch physical plan\"\"\"\n      Log.info(\"Watch triggered for topology pplan: \" + topologyName)\n      topology.set_physical_plan(data)\n      if not data:\n        Log.debug(\"No data to be set\")\n\n    def on_topology_packing_plan(data):\n      \"\"\"watch packing plan\"\"\"\n      Log.info(\"Watch triggered for topology packing plan: \" + topologyName)\n      topology.set_packing_plan(data)\n      if not data:\n        Log.debug(\"No data to be set\")\n\n    def on_topology_execution_state(data):\n      \"\"\"watch execution state\"\"\"\n      Log.info(\"Watch triggered for topology execution state: \" + topologyName)\n      topology.set_execution_state(data)\n      if not data:\n        Log.debug(\"No data to be set\")\n\n    def on_topology_tmaster(data):\n      \"\"\"set tmaster\"\"\"\n      Log.info(\"Watch triggered for topology tmaster: \" + topologyName)\n      topology.set_tmaster(data)\n      if not data:\n        Log.debug(\"No data to be set\")\n\n    def on_topology_scheduler_location(data):\n      \"\"\"set scheduler location\"\"\"\n      Log.info(\"Watch triggered for topology scheduler location: \" + topologyName)\n      topology.set_scheduler_location(data)\n      if not data:\n        Log.debug(\"No data to be set\")\n\n    # Set watches on the pplan, execution_state, tmaster and scheduler_location.\n    state_manager.get_pplan(topologyName, on_topology_pplan)\n    state_manager.get_packing_plan(topologyName, on_topology_packing_plan)\n    state_manager.get_execution_state(topologyName, on_topology_execution_state)\n    state_manager.get_tmaster(topologyName, on_topology_tmaster)\n    state_manager.get_scheduler_location(topologyName, on_topology_scheduler_location)", "response": "Adds a new topology to the local cache and sets a watchon on any changes on the topology."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef removeTopology(self, topology_name, state_manager_name):\n    topologies = []\n    for top in self.topologies:\n      if (top.name == topology_name and\n          top.state_manager_name == state_manager_name):\n        # Remove topologyInfo\n        if (topology_name, state_manager_name) in self.topologyInfos:\n          self.topologyInfos.pop((topology_name, state_manager_name))\n      else:\n        topologies.append(top)\n\n    self.topologies = topologies", "response": "Removes the topology from the local cache."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef extract_execution_state(self, topology):\n    execution_state = topology.execution_state\n\n    executionState = {\n        \"cluster\": execution_state.cluster,\n        \"environ\": execution_state.environ,\n        \"role\": execution_state.role,\n        \"jobname\": topology.name,\n        \"submission_time\": execution_state.submission_time,\n        \"submission_user\": execution_state.submission_user,\n        \"release_username\": execution_state.release_state.release_username,\n        \"release_tag\": execution_state.release_state.release_tag,\n        \"release_version\": execution_state.release_state.release_version,\n        \"has_physical_plan\": None,\n        \"has_tmaster_location\": None,\n        \"has_scheduler_location\": None,\n        \"extra_links\": [],\n    }\n\n    for extra_link in self.config.extra_links:\n      link = extra_link.copy()\n      link[\"url\"] = self.config.get_formatted_url(executionState,\n                                                  link[EXTRA_LINK_FORMATTER_KEY])\n      executionState[\"extra_links\"].append(link)\n    return executionState", "response": "Extracts the execution state of a topology from the Tracker."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nextract the scheduler location from the given topology", "response": "def extract_scheduler_location(self, topology):\n    \"\"\"\n    Returns the representation of scheduler location that will\n    be returned from Tracker.\n    \"\"\"\n    schedulerLocation = {\n        \"name\": None,\n        \"http_endpoint\": None,\n        \"job_page_link\": None,\n    }\n\n    if topology.scheduler_location:\n      schedulerLocation[\"name\"] = topology.scheduler_location.topology_name\n      schedulerLocation[\"http_endpoint\"] = topology.scheduler_location.http_endpoint\n      schedulerLocation[\"job_page_link\"] = \\\n          topology.scheduler_location.job_page_link[0] \\\n          if len(topology.scheduler_location.job_page_link) > 0 else \"\"\n\n    return schedulerLocation"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef extract_tmaster(self, topology):\n    tmasterLocation = {\n        \"name\": None,\n        \"id\": None,\n        \"host\": None,\n        \"controller_port\": None,\n        \"master_port\": None,\n        \"stats_port\": None,\n    }\n    if topology.tmaster:\n      tmasterLocation[\"name\"] = topology.tmaster.topology_name\n      tmasterLocation[\"id\"] = topology.tmaster.topology_id\n      tmasterLocation[\"host\"] = topology.tmaster.host\n      tmasterLocation[\"controller_port\"] = topology.tmaster.controller_port\n      tmasterLocation[\"master_port\"] = topology.tmaster.master_port\n      tmasterLocation[\"stats_port\"] = topology.tmaster.stats_port\n\n    return tmasterLocation", "response": "Returns the representation of tmaster that will\n    be returned from Tracker."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the representation of the logical plan that will be returned from Tracker.", "response": "def extract_logical_plan(self, topology):\n    \"\"\"\n    Returns the representation of logical plan that will\n    be returned from Tracker.\n    \"\"\"\n    logicalPlan = {\n        \"spouts\": {},\n        \"bolts\": {},\n    }\n\n    # Add spouts.\n    for spout in topology.spouts():\n      spoutName = spout.comp.name\n      spoutType = \"default\"\n      spoutSource = \"NA\"\n      spoutVersion = \"NA\"\n      spoutConfigs = spout.comp.config.kvs\n      for kvs in spoutConfigs:\n        if kvs.key == \"spout.type\":\n          spoutType = javaobj.loads(kvs.serialized_value)\n        elif kvs.key == \"spout.source\":\n          spoutSource = javaobj.loads(kvs.serialized_value)\n        elif kvs.key == \"spout.version\":\n          spoutVersion = javaobj.loads(kvs.serialized_value)\n      spoutPlan = {\n          \"config\": convert_pb_kvs(spoutConfigs, include_non_primitives=False),\n          \"type\": spoutType,\n          \"source\": spoutSource,\n          \"version\": spoutVersion,\n          \"outputs\": []\n      }\n      for outputStream in list(spout.outputs):\n        spoutPlan[\"outputs\"].append({\n            \"stream_name\": outputStream.stream.id\n        })\n\n      logicalPlan[\"spouts\"][spoutName] = spoutPlan\n\n    # Add bolts.\n    for bolt in topology.bolts():\n      boltName = bolt.comp.name\n      boltPlan = {\n          \"config\": convert_pb_kvs(bolt.comp.config.kvs, include_non_primitives=False),\n          \"outputs\": [],\n          \"inputs\": []\n      }\n      for outputStream in list(bolt.outputs):\n        boltPlan[\"outputs\"].append({\n            \"stream_name\": outputStream.stream.id\n        })\n      for inputStream in list(bolt.inputs):\n        boltPlan[\"inputs\"].append({\n            \"stream_name\": inputStream.stream.id,\n            \"component_name\": inputStream.stream.component_name,\n            \"grouping\": topology_pb2.Grouping.Name(inputStream.gtype)\n        })\n\n      logicalPlan[\"bolts\"][boltName] = boltPlan\n\n    return logicalPlan"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the representation of the physical plan that will be returned from Tracker.", "response": "def extract_physical_plan(self, topology):\n    \"\"\"\n    Returns the representation of physical plan that will\n    be returned from Tracker.\n    \"\"\"\n    physicalPlan = {\n        \"instances\": {},\n        \"instance_groups\": {},\n        \"stmgrs\": {},\n        \"spouts\": {},\n        \"bolts\": {},\n        \"config\": {},\n        \"components\": {}\n    }\n\n    if not topology.physical_plan:\n      return physicalPlan\n\n    spouts = topology.spouts()\n    bolts = topology.bolts()\n    stmgrs = None\n    instances = None\n\n    # Physical Plan\n    stmgrs = list(topology.physical_plan.stmgrs)\n    instances = list(topology.physical_plan.instances)\n\n    # Configs\n    if topology.physical_plan.topology.topology_config:\n      physicalPlan[\"config\"] = convert_pb_kvs(topology.physical_plan.topology.topology_config.kvs)\n\n    for spout in spouts:\n      spout_name = spout.comp.name\n      physicalPlan[\"spouts\"][spout_name] = []\n      if spout_name not in physicalPlan[\"components\"]:\n        physicalPlan[\"components\"][spout_name] = {\n            \"config\": convert_pb_kvs(spout.comp.config.kvs)\n        }\n    for bolt in bolts:\n      bolt_name = bolt.comp.name\n      physicalPlan[\"bolts\"][bolt_name] = []\n      if bolt_name not in physicalPlan[\"components\"]:\n        physicalPlan[\"components\"][bolt_name] = {\n            \"config\": convert_pb_kvs(bolt.comp.config.kvs)\n        }\n\n    for stmgr in stmgrs:\n      host = stmgr.host_name\n      cwd = stmgr.cwd\n      shell_port = stmgr.shell_port if stmgr.HasField(\"shell_port\") else None\n      physicalPlan[\"stmgrs\"][stmgr.id] = {\n          \"id\": stmgr.id,\n          \"host\": host,\n          \"port\": stmgr.data_port,\n          \"shell_port\": shell_port,\n          \"cwd\": cwd,\n          \"pid\": stmgr.pid,\n          \"joburl\": utils.make_shell_job_url(host, shell_port, cwd),\n          \"logfiles\": utils.make_shell_logfiles_url(host, shell_port, cwd),\n          \"instance_ids\": []\n      }\n\n    instance_groups = collections.OrderedDict()\n    for instance in instances:\n      instance_id = instance.instance_id\n      stmgrId = instance.stmgr_id\n      name = instance.info.component_name\n      stmgrInfo = physicalPlan[\"stmgrs\"][stmgrId]\n      host = stmgrInfo[\"host\"]\n      cwd = stmgrInfo[\"cwd\"]\n      shell_port = stmgrInfo[\"shell_port\"]\n\n\n      # instance_id format container_<index>_component_1\n      # group name is container_<index>\n      group_name = instance_id.rsplit(\"_\", 2)[0]\n      igroup = instance_groups.get(group_name, list())\n      igroup.append(instance_id)\n      instance_groups[group_name] = igroup\n\n      physicalPlan[\"instances\"][instance_id] = {\n          \"id\": instance_id,\n          \"name\": name,\n          \"stmgrId\": stmgrId,\n          \"logfile\": utils.make_shell_logfiles_url(host, shell_port, cwd, instance.instance_id),\n      }\n      physicalPlan[\"stmgrs\"][stmgrId][\"instance_ids\"].append(instance_id)\n      if name in physicalPlan[\"spouts\"]:\n        physicalPlan[\"spouts\"][name].append(instance_id)\n      else:\n        physicalPlan[\"bolts\"][name].append(instance_id)\n\n    physicalPlan[\"instance_groups\"] = instance_groups\n\n    return physicalPlan"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nextracts the packing plan that will be returned from Tracker.", "response": "def extract_packing_plan(self, topology):\n    \"\"\"\n    Returns the representation of packing plan that will\n    be returned from Tracker.\n    \"\"\"\n    packingPlan = {\n        \"id\": \"\",\n        \"container_plans\": []\n    }\n\n    if not topology.packing_plan:\n      return packingPlan\n\n    container_plans = topology.packing_plan.container_plans\n\n    containers = []\n    for container_plan in container_plans:\n      instances = []\n      for instance_plan in container_plan.instance_plans:\n        instance_resources = {\"cpu\": instance_plan.resource.cpu,\n                              \"ram\": instance_plan.resource.ram,\n                              \"disk\": instance_plan.resource.disk}\n        instance = {\"component_name\" : instance_plan.component_name,\n                    \"task_id\" : instance_plan.task_id,\n                    \"component_index\": instance_plan.component_index,\n                    \"instance_resources\": instance_resources}\n        instances.append(instance)\n      required_resource = {\"cpu\": container_plan.requiredResource.cpu,\n                           \"ram\": container_plan.requiredResource.ram,\n                           \"disk\": container_plan.requiredResource.disk}\n      scheduled_resource = {}\n      if container_plan.scheduledResource:\n        scheduled_resource = {\"cpu\": container_plan.scheduledResource.cpu,\n                              \"ram\": container_plan.scheduledResource.ram,\n                              \"disk\": container_plan.scheduledResource.disk}\n      container = {\"id\": container_plan.id,\n                   \"instances\": instances,\n                   \"required_resources\": required_resource,\n                   \"scheduled_resources\": scheduled_resource}\n      containers.append(container)\n\n    packingPlan[\"id\"] = topology.packing_plan.id\n    packingPlan[\"container_plans\"] = containers\n    return json.dumps(packingPlan)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the topology info for the given topology.", "response": "def setTopologyInfo(self, topology):\n    \"\"\"\n    Extracts info from the stored proto states and\n    convert it into representation that is exposed using\n    the API.\n    This method is called on any change for the topology.\n    For example, when a container moves and its host or some\n    port changes. All the information is parsed all over\n    again and cache is updated.\n    \"\"\"\n    # Execution state is the most basic info.\n    # If there is no execution state, just return\n    # as the rest of the things don't matter.\n    if not topology.execution_state:\n      Log.info(\"No execution state found for: \" + topology.name)\n      return\n\n    Log.info(\"Setting topology info for topology: \" + topology.name)\n    has_physical_plan = True\n    if not topology.physical_plan:\n      has_physical_plan = False\n\n    Log.info(\"Setting topology info for topology: \" + topology.name)\n    has_packing_plan = True\n    if not topology.packing_plan:\n      has_packing_plan = False\n\n    has_tmaster_location = True\n    if not topology.tmaster:\n      has_tmaster_location = False\n\n    has_scheduler_location = True\n    if not topology.scheduler_location:\n      has_scheduler_location = False\n\n    topologyInfo = {\n        \"name\": topology.name,\n        \"id\": topology.id,\n        \"logical_plan\": None,\n        \"physical_plan\": None,\n        \"packing_plan\": None,\n        \"execution_state\": None,\n        \"tmaster_location\": None,\n        \"scheduler_location\": None,\n    }\n\n    executionState = self.extract_execution_state(topology)\n    executionState[\"has_physical_plan\"] = has_physical_plan\n    executionState[\"has_packing_plan\"] = has_packing_plan\n    executionState[\"has_tmaster_location\"] = has_tmaster_location\n    executionState[\"has_scheduler_location\"] = has_scheduler_location\n    executionState[\"status\"] = topology.get_status()\n\n    topologyInfo[\"metadata\"] = self.extract_metadata(topology)\n    topologyInfo[\"runtime_state\"] = self.extract_runtime_state(topology)\n\n    topologyInfo[\"execution_state\"] = executionState\n    topologyInfo[\"logical_plan\"] = self.extract_logical_plan(topology)\n    topologyInfo[\"physical_plan\"] = self.extract_physical_plan(topology)\n    topologyInfo[\"packing_plan\"] = self.extract_packing_plan(topology)\n    topologyInfo[\"tmaster_location\"] = self.extract_tmaster(topology)\n    topologyInfo[\"scheduler_location\"] = self.extract_scheduler_location(topology)\n\n    self.topologyInfos[(topology.name, topology.state_manager_name)] = topologyInfo"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef getTopologyInfo(self, topologyName, cluster, role, environ):\n    # Iterate over the values to filter the desired topology.\n    for (topology_name, _), topologyInfo in self.topologyInfos.items():\n      executionState = topologyInfo[\"execution_state\"]\n      if (topologyName == topology_name and\n          cluster == executionState[\"cluster\"] and\n          environ == executionState[\"environ\"]):\n        # If role is specified, first try to match \"role\" field. If \"role\" field\n        # does not exist, try to match \"submission_user\" field.\n        if not role or executionState.get(\"role\") == role:\n          return topologyInfo\n    if role is not None:\n      Log.info(\"Could not find topology info for topology: %s,\" \\\n               \"cluster: %s, role: %s, and environ: %s\",\n               topologyName, cluster, role, environ)\n    else:\n      Log.info(\"Could not find topology info for topology: %s,\" \\\n               \"cluster: %s and environ: %s\", topologyName, cluster, environ)\n    raise Exception(\"No topology found\")", "response": "Returns the JSON representation of a topologyInfo object by its name cluster environ and role."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef validated_formatter(self, url_format):\n    # We try to create a string by substituting all known\n    # parameters. If an unknown parameter is present, an error\n    # will be thrown\n    valid_parameters = {\n        \"${CLUSTER}\": \"cluster\",\n        \"${ENVIRON}\": \"environ\",\n        \"${TOPOLOGY}\": \"topology\",\n        \"${ROLE}\": \"role\",\n        \"${USER}\": \"user\",\n    }\n    dummy_formatted_url = url_format\n    for key, value in valid_parameters.items():\n      dummy_formatted_url = dummy_formatted_url.replace(key, value)\n\n    # All $ signs must have been replaced\n    if '$' in dummy_formatted_url:\n      raise Exception(\"Invalid viz.url.format: %s\" % (url_format))\n\n    # No error is thrown, so the format is valid.\n    return url_format", "response": "validate visualization url format"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef emit(self, tup, tup_id=None, stream=Stream.DEFAULT_STREAM_ID,\n           direct_task=None, need_task_ids=False):\n    \"\"\"Emits a new tuple from this Spout\n\n    It is compatible with StreamParse API.\n\n    :type tup: list or tuple\n    :param tup: the new output Tuple to send from this spout,\n                should contain only serializable data.\n    :type tup_id: str or object\n    :param tup_id: the ID for the Tuple. Leave this blank for an unreliable emit.\n                   (Same as messageId in Java)\n    :type stream: str\n    :param stream: the ID of the stream this Tuple should be emitted to.\n                   Leave empty to emit to the default stream.\n    :type direct_task: int\n    :param direct_task: the task to send the Tuple to if performing a direct emit.\n    :type need_task_ids: bool\n    :param need_task_ids: indicate whether or not you would like the task IDs the Tuple was emitted.\n    \"\"\"\n    # first check whether this tuple is sane\n    self.pplan_helper.check_output_schema(stream, tup)\n\n    # get custom grouping target task ids; get empty list if not custom grouping\n    custom_target_task_ids = self.pplan_helper.choose_tasks_for_custom_grouping(stream, tup)\n\n    self.pplan_helper.context.invoke_hook_emit(tup, stream, None)\n\n    data_tuple = tuple_pb2.HeronDataTuple()\n    data_tuple.key = 0\n\n    if direct_task is not None:\n      if not isinstance(direct_task, int):\n        raise TypeError(\"direct_task argument needs to be an integer, given: %s\"\n                        % str(type(direct_task)))\n      # performing emit-direct\n      data_tuple.dest_task_ids.append(direct_task)\n    elif custom_target_task_ids is not None:\n      # for custom grouping\n      for task_id in custom_target_task_ids:\n        data_tuple.dest_task_ids.append(task_id)\n\n    if tup_id is not None:\n      tuple_info = TupleHelper.make_root_tuple_info(stream, tup_id)\n      if self.acking_enabled:\n        # this message is rooted\n        root = data_tuple.roots.add()\n        root.taskid = self.pplan_helper.my_task_id\n        root.key = tuple_info.key\n        self.in_flight_tuples[tuple_info.key] = tuple_info\n      else:\n        self.immediate_acks.append(tuple_info)\n\n    tuple_size_in_bytes = 0\n\n    start_time = time.time()\n\n    # Serialize\n    for obj in tup:\n      serialized = self.serializer.serialize(obj)\n      data_tuple.values.append(serialized)\n      tuple_size_in_bytes += len(serialized)\n\n    serialize_latency_ns = (time.time() - start_time) * system_constants.SEC_TO_NS\n    self.spout_metrics.serialize_data_tuple(stream, serialize_latency_ns)\n\n    super(SpoutInstance, self).admit_data_tuple(stream_id=stream, data_tuple=data_tuple,\n                                                tuple_size_in_bytes=tuple_size_in_bytes)\n    self.total_tuples_emitted += 1\n    self.spout_metrics.update_emit_count(stream)\n    if need_task_ids:\n      sent_task_ids = custom_target_task_ids or []\n      if direct_task is not None:\n        sent_task_ids.append(direct_task)\n      return sent_task_ids", "response": "Emits a new tuple from this Spout."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _is_continue_to_work(self):\n    if not self._is_topology_running():\n      return False\n\n    max_spout_pending = \\\n      self.pplan_helper.context.get_cluster_config().get(api_constants.TOPOLOGY_MAX_SPOUT_PENDING)\n\n    if not self.acking_enabled and self.output_helper.is_out_queue_available():\n      return True\n    elif self.acking_enabled and self.output_helper.is_out_queue_available() and \\\n        len(self.in_flight_tuples) < max_spout_pending:\n      return True\n    elif self.acking_enabled and not self.in_stream.is_empty():\n      return True\n    else:\n      return False", "response": "Checks whether we need to continue to work based on the current state of the topology."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating parser for topology - name", "response": "def create_parser(subparsers):\n  \"\"\" create parser \"\"\"\n  components_parser = subparsers.add_parser(\n      'components',\n      help='Display information of a topology\\'s components',\n      usage=\"%(prog)s cluster/[role]/[env] topology-name [options]\",\n      add_help=False)\n  args.add_cluster_role_env(components_parser)\n  args.add_topology_name(components_parser)\n  args.add_spouts(components_parser)\n  args.add_bolts(components_parser)\n  args.add_verbose(components_parser)\n  args.add_tracker_url(components_parser)\n  args.add_config(components_parser)\n  components_parser.set_defaults(subcommand='components')\n\n  return subparsers"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef to_table(components, topo_info):\n  inputs, outputs = defaultdict(list), defaultdict(list)\n  for ctype, component in components.items():\n    if ctype == 'bolts':\n      for component_name, component_info in component.items():\n        for input_stream in component_info['inputs']:\n          input_name = input_stream['component_name']\n          inputs[component_name].append(input_name)\n          outputs[input_name].append(component_name)\n  info = []\n  spouts_instance = topo_info['physical_plan']['spouts']\n  bolts_instance = topo_info['physical_plan']['bolts']\n  for ctype, component in components.items():\n    # stages is an int so keep going\n    if ctype == \"stages\":\n      continue\n    for component_name, component_info in component.items():\n      row = [ctype[:-1], component_name]\n      if ctype == 'spouts':\n        row.append(len(spouts_instance[component_name]))\n      else:\n        row.append(len(bolts_instance[component_name]))\n      row.append(','.join(inputs.get(component_name, ['-'])))\n      row.append(','.join(outputs.get(component_name, ['-'])))\n      info.append(row)\n  header = ['type', 'name', 'parallelism', 'input', 'output']\n  return info, header", "response": "normalize raw logical plan info to table"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef filter_bolts(table, header):\n  bolts_info = []\n  for row in table:\n    if row[0] == 'bolt':\n      bolts_info.append(row)\n  return bolts_info, header", "response": "filter to keep bolts"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfiltering to keep spouts", "response": "def filter_spouts(table, header):\n  \"\"\" filter to keep spouts \"\"\"\n  spouts_info = []\n  for row in table:\n    if row[0] == 'spout':\n      spouts_info.append(row)\n  return spouts_info, header"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting the topologies of the current node", "response": "def get_topologies(self, callback=None):\n    \"\"\" get topologies \"\"\"\n    isWatching = False\n\n    # Temp dict used to return result\n    # if callback is not provided.\n    ret = {\n        \"result\": None\n    }\n    if callback:\n      isWatching = True\n    else:\n      def callback(data):\n        \"\"\"Custom callback to get the topologies right now.\"\"\"\n        ret[\"result\"] = data\n\n    try:\n      # Ensure the topology path exists. If a topology has never been deployed\n      # then the path will not exist so create it and don't crash.\n      # (fixme) add a watch instead of creating the path?\n      self.client.ensure_path(self.get_topologies_path())\n\n      self._get_topologies_with_watch(callback, isWatching)\n    except NoNodeError:\n      self.client.stop()\n      path = self.get_topologies_path()\n      raise_(StateException(\"Error required topology path '%s' not found\" % (path),\n                            StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2])\n\n    # The topologies are now populated with the data.\n    return ret[\"result\"]"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_topology(self, topologyName, callback=None):\n    isWatching = False\n\n    # Temp dict used to return result\n    # if callback is not provided.\n    ret = {\n        \"result\": None\n    }\n    if callback:\n      isWatching = True\n    else:\n      def callback(data):\n        \"\"\"Custom callback to get the topologies right now.\"\"\"\n        ret[\"result\"] = data\n\n    self._get_topology_with_watch(topologyName, callback, isWatching)\n\n    # The topologies are now populated with the data.\n    return ret[\"result\"]", "response": "get the topology with the given name"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _get_topology_with_watch(self, topologyName, callback, isWatching):\n    path = self.get_topology_path(topologyName)\n    if isWatching:\n      LOG.info(\"Adding data watch for path: \" + path)\n\n    # pylint: disable=unused-variable, unused-argument\n    @self.client.DataWatch(path)\n    def watch_topology(data, stats):\n      \"\"\" watch topology \"\"\"\n      if data:\n        topology = Topology()\n        topology.ParseFromString(data)\n        callback(topology)\n      else:\n        callback(None)\n\n      # Returning False will result in no future watches\n      # being triggered. If isWatching is True, then\n      # the future watches will be triggered.\n      return isWatching", "response": "Helper function to get topology with a callback."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndelete a topology from the zookeeper.", "response": "def delete_topology(self, topologyName):\n    \"\"\" delete topology \"\"\"\n    path = self.get_topology_path(topologyName)\n    LOG.info(\"Removing topology: {0} from path: {1}\".format(\n        topologyName, path))\n    try:\n      self.client.delete(path)\n      return True\n    except NoNodeError:\n      raise_(StateException(\"NoNodeError while deteling topology\",\n                            StateException.EX_TYPE_NO_NODE_ERROR), sys.exc_info()[2])\n    except NotEmptyError:\n      raise_(StateException(\"NotEmptyError while deleting topology\",\n                            StateException.EX_TYPE_NOT_EMPTY_ERROR), sys.exc_info()[2])\n    except ZookeeperError:\n      raise_(StateException(\"Zookeeper while deleting topology\",\n                            StateException.EX_TYPE_ZOOKEEPER_ERROR), sys.exc_info()[2])\n    except Exception:\n      # Just re raise the exception.\n      raise"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_packing_plan(self, topologyName, callback=None):\n    isWatching = False\n\n    # Temp dict used to return result\n    # if callback is not provided.\n    ret = {\n        \"result\": None\n    }\n    if callback:\n      isWatching = True\n    else:\n      def callback(data):\n        \"\"\" Custom callback to get the topologies right now. \"\"\"\n        ret[\"result\"] = data\n\n    self._get_packing_plan_with_watch(topologyName, callback, isWatching)\n\n    # The topologies are now populated with the data.\n    return ret[\"result\"]", "response": "get packing plan for a given topology"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _get_packing_plan_with_watch(self, topologyName, callback, isWatching):\n    path = self.get_packing_plan_path(topologyName)\n    if isWatching:\n      LOG.info(\"Adding data watch for path: \" + path)\n\n    # pylint: disable=unused-argument,unused-variable\n    @self.client.DataWatch(path)\n    def watch_packing_plan(data, stats):\n      \"\"\" watch the packing plan for updates \"\"\"\n      if data:\n        packing_plan = PackingPlan()\n        packing_plan.ParseFromString(data)\n        callback(packing_plan)\n      else:\n        callback(None)\n\n      # Returning False will result in no future watches\n      # being triggered. If isWatching is True, then\n      # the future watches will be triggered.\n      return isWatching", "response": "Helper function to get packing_plan with a callback."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_pplan(self, topologyName, callback=None):\n    isWatching = False\n\n    # Temp dict used to return result\n    # if callback is not provided.\n    ret = {\n        \"result\": None\n    }\n    if callback:\n      isWatching = True\n    else:\n      def callback(data):\n        \"\"\"\n        Custom callback to get the topologies right now.\n        \"\"\"\n        ret[\"result\"] = data\n\n    self._get_pplan_with_watch(topologyName, callback, isWatching)\n\n    # The topologies are now populated with the data.\n    return ret[\"result\"]", "response": "get physical plan for a given topologyName"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the execution state of a topology", "response": "def get_execution_state(self, topologyName, callback=None):\n    \"\"\" get execution state \"\"\"\n    isWatching = False\n\n    # Temp dict used to return result\n    # if callback is not provided.\n    ret = {\n        \"result\": None\n    }\n    if callback:\n      isWatching = True\n    else:\n      def callback(data):\n        \"\"\"\n        Custom callback to get the topologies right now.\n        \"\"\"\n        ret[\"result\"] = data\n\n    self._get_execution_state_with_watch(topologyName, callback, isWatching)\n\n    # The topologies are now populated with the data.\n    return ret[\"result\"]"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _get_execution_state_with_watch(self, topologyName, callback, isWatching):\n    path = self.get_execution_state_path(topologyName)\n    if isWatching:\n      LOG.info(\"Adding data watch for path: \" + path)\n\n    # pylint: disable=unused-variable, unused-argument\n    @self.client.DataWatch(path)\n    def watch_execution_state(data, stats):\n      \"\"\" invoke callback to watch execute state \"\"\"\n      if data:\n        executionState = ExecutionState()\n        executionState.ParseFromString(data)\n        callback(executionState)\n      else:\n        callback(None)\n\n      # Returning False will result in no future watches\n      # being triggered. If isWatching is True, then\n      # the future watches will be triggered.\n      return isWatching", "response": "Helper function to get execution state with a callback."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_tmaster(self, topologyName, callback=None):\n    isWatching = False\n\n    # Temp dict used to return result\n    # if callback is not provided.\n    ret = {\n        \"result\": None\n    }\n    if callback:\n      isWatching = True\n    else:\n      def callback(data):\n        \"\"\"\n        Custom callback to get the topologies right now.\n        \"\"\"\n        ret[\"result\"] = data\n\n    self._get_tmaster_with_watch(topologyName, callback, isWatching)\n\n    # The topologies are now populated with the data.\n    return ret[\"result\"]", "response": "get tmaster for a given topologyName"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _get_scheduler_location_with_watch(self, topologyName, callback, isWatching):\n    path = self.get_scheduler_location_path(topologyName)\n    if isWatching:\n      LOG.info(\"Adding data watch for path: \" + path)\n\n    # pylint: disable=unused-variable, unused-argument\n    @self.client.DataWatch(path)\n    def watch_scheduler_location(data, stats):\n      \"\"\" invoke callback to watch scheduler location \"\"\"\n      if data:\n        scheduler_location = SchedulerLocation()\n        scheduler_location.ParseFromString(data)\n        callback(scheduler_location)\n      else:\n        callback(None)\n\n      # Returning False will result in no future watches\n      # being triggered. If isWatching is True, then\n      # the future watches will be triggered.\n      return isWatching", "response": "Helper function to get scheduler location with a callback."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef readObject(self):\n    try:\n      _, res = self._read_and_exec_opcode(ident=0)\n\n      position_bak = self.object_stream.tell()\n      the_rest = self.object_stream.read()\n      if len(the_rest):\n        log_error(\"Warning!!!!: Stream still has %s bytes left.\\\nEnable debug mode of logging to see the hexdump.\" % len(the_rest))\n        log_debug(self._create_hexdump(the_rest))\n      else:\n        log_debug(\"Java Object unmarshalled succesfully!\")\n      self.object_stream.seek(position_bak)\n\n      return res\n    except Exception:\n      self._oops_dump_state()\n      raise", "response": "read object from the object stream"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef getMetricsTimeline(tmaster,\n                       component_name,\n                       metric_names,\n                       instances,\n                       start_time,\n                       end_time,\n                       callback=None):\n  \"\"\"\n  Get the specified metrics for the given component name of this topology.\n  Returns the following dict on success:\n  {\n    \"timeline\": {\n      <metricname>: {\n        <instance>: {\n          <start_time> : <numeric value>,\n          <start_time> : <numeric value>,\n          ...\n        }\n        ...\n      }, ...\n    },\n    \"starttime\": <numeric value>,\n    \"endtime\": <numeric value>,\n    \"component\": \"...\"\n  }\n\n  Returns the following dict on failure:\n  {\n    \"message\": \"...\"\n  }\n  \"\"\"\n  # Tmaster is the proto object and must have host and port for stats.\n  if not tmaster or not tmaster.host or not tmaster.stats_port:\n    raise Exception(\"No Tmaster found\")\n\n  host = tmaster.host\n  port = tmaster.stats_port\n\n  # Create the proto request object to get metrics.\n\n  metricRequest = tmaster_pb2.MetricRequest()\n  metricRequest.component_name = component_name\n\n  # If no instances are give, metrics for all instances\n  # are fetched by default.\n  if len(instances) > 0:\n    for instance in instances:\n      metricRequest.instance_id.append(instance)\n\n  for metricName in metric_names:\n    metricRequest.metric.append(metricName)\n\n  metricRequest.explicit_interval.start = start_time\n  metricRequest.explicit_interval.end = end_time\n  metricRequest.minutely = True\n\n  # Serialize the metricRequest to send as a payload\n  # with the HTTP request.\n  metricRequestString = metricRequest.SerializeToString()\n\n  # Form and send the http request.\n  url = \"http://{0}:{1}/stats\".format(host, port)\n  request = tornado.httpclient.HTTPRequest(url,\n                                           body=metricRequestString,\n                                           method='POST',\n                                           request_timeout=5)\n\n  Log.debug(\"Making HTTP call to fetch metrics\")\n  Log.debug(\"url: \" + url)\n  try:\n    client = tornado.httpclient.AsyncHTTPClient()\n    result = yield client.fetch(request)\n    Log.debug(\"HTTP call complete.\")\n  except tornado.httpclient.HTTPError as e:\n    raise Exception(str(e))\n\n\n  # Check the response code - error if it is in 400s or 500s\n  responseCode = result.code\n  if responseCode >= 400:\n    message = \"Error in getting metrics from Tmaster, code: \" + responseCode\n    Log.error(message)\n    raise Exception(message)\n\n  # Parse the response from tmaster.\n  metricResponse = tmaster_pb2.MetricResponse()\n  metricResponse.ParseFromString(result.body)\n\n  if metricResponse.status.status == common_pb2.NOTOK:\n    if metricResponse.status.HasField(\"message\"):\n      Log.warn(\"Received response from Tmaster: %s\", metricResponse.status.message)\n\n  # Form the response.\n  ret = {}\n  ret[\"starttime\"] = start_time\n  ret[\"endtime\"] = end_time\n  ret[\"component\"] = component_name\n  ret[\"timeline\"] = {}\n\n  # Loop through all the metrics\n  # One instance corresponds to one metric, which can have\n  # multiple IndividualMetrics for each metricname requested.\n  for metric in metricResponse.metric:\n    instance = metric.instance_id\n\n    # Loop through all individual metrics.\n    for im in metric.metric:\n      metricname = im.name\n      if metricname not in ret[\"timeline\"]:\n        ret[\"timeline\"][metricname] = {}\n      if instance not in ret[\"timeline\"][metricname]:\n        ret[\"timeline\"][metricname][instance] = {}\n\n      # We get minutely metrics.\n      # Interval-values correspond to the minutely mark for which\n      # this metric value corresponds to.\n      for interval_value in im.interval_values:\n        ret[\"timeline\"][metricname][instance][interval_value.interval.start] = interval_value.value\n\n  raise tornado.gen.Return(ret)", "response": "Get the specified metrics for the given component name and instances."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_parser(subparsers):\n  '''\n  :param subparsers:\n  :return:\n  '''\n  parser = subparsers.add_parser(\n      'version',\n      help='Print version of heron-cli',\n      usage=\"%(prog)s [options] [cluster]\",\n      add_help=True)\n\n  add_version_titles(parser)\n\n  parser.add_argument(\n      'cluster',\n      nargs='?',\n      type=str,\n      default=\"\",\n      help='Name of the cluster')\n\n  cli_args.add_service_url(parser)\n\n  parser.set_defaults(subcommand='version')\n  return parser", "response": "Create a parser for heron - cli"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef run(command, parser, cl_args, unknown_args):\n  '''\n  :param command:\n  :param parser:\n  :param args:\n  :param unknown_args:\n  :return:\n  '''\n  cluster = cl_args['cluster']\n\n  # server mode\n  if cluster:\n    config_file = config.heron_rc_file()\n    client_confs = dict()\n\n    # Read the cluster definition, if not found\n    client_confs = cdefs.read_server_mode_cluster_definition(cluster, cl_args, config_file)\n\n    if not client_confs[cluster]:\n      Log.error('Neither service url nor %s cluster definition in %s file', cluster, config_file)\n      return SimpleResult(Status.HeronError)\n\n    # if cluster definition exists, but service_url is not set, it is an error\n    if not 'service_url' in client_confs[cluster]:\n      Log.error('No service url for %s cluster in %s', cluster, config_file)\n      sys.exit(1)\n\n    service_endpoint = cl_args['service_url']\n    service_apiurl = service_endpoint + rest.ROUTE_SIGNATURES[command][1]\n    service_method = rest.ROUTE_SIGNATURES[command][0]\n\n    try:\n      r = service_method(service_apiurl)\n      if r.status_code != requests.codes.ok:\n        Log.error(r.json().get('message', \"Unknown error from API server %d\" % r.status_code))\n      sorted_items = sorted(r.json().items(), key=lambda tup: tup[0])\n      for key, value in sorted_items:\n        print(\"%s : %s\" % (key, value))\n    except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:\n      Log.error(err)\n      return SimpleResult(Status.HeronError)\n  else:\n    config.print_build_info()\n\n  return SimpleResult(Status.Ok)", "response": "This function is used to run the command line interface."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nvalidating that the names of all state locations are unique.", "response": "def validate_state_locations(self):\n    \"\"\"\n    Names of all state locations must be unique.\n    \"\"\"\n    names = map(lambda loc: loc[\"name\"], self.locations)\n    assert len(names) == len(set(names)), \"Names of state locations must be unique\""}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nadd the arguments to the parser object.", "response": "def add_arguments(parser):\n  '''\n  :param parser:\n  :return:\n  '''\n  parser.add_argument(\n      '--tracker_url',\n      metavar='(a url; path to tracker; default: \"' + consts.DEFAULT_TRACKER_URL + '\")',\n      default=consts.DEFAULT_TRACKER_URL)\n\n  parser.add_argument(\n      '--address',\n      metavar='(an string; address to listen; default: \"' + consts.DEFAULT_ADDRESS + '\")',\n      default=consts.DEFAULT_ADDRESS)\n\n  parser.add_argument(\n      '--port',\n      metavar='(an integer; port to listen; default: ' + str(consts.DEFAULT_PORT) + ')',\n      type=int,\n      default=consts.DEFAULT_PORT)\n\n  parser.add_argument(\n      '--base_url',\n      metavar='(a string; the base url path if operating behind proxy; default: '\n      + str(consts.DEFAULT_BASE_URL) + ')',\n      default=consts.DEFAULT_BASE_URL)\n\n  return parser"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get(self):\n    '''\n    :return:\n    '''\n    cluster = self.get_argument(\"cluster\")\n    environ = self.get_argument(\"environ\")\n    topology = self.get_argument(\"topology\")\n    component = self.get_argument(\"component\", default=None)\n    metricnames = self.get_arguments(\"metricname\")\n    instances = self.get_arguments(\"instance\")\n    interval = self.get_argument(\"interval\", default=-1)\n    time_range = (0, interval)\n    compnames = [component] if component else (yield access.get_comps(cluster, environ, topology))\n\n    # fetch the metrics\n    futures = {}\n    for comp in compnames:\n      future = access.get_comp_metrics(\n          cluster, environ, topology, comp, instances,\n          metricnames, time_range)\n      futures[comp] = future\n\n    results = yield futures\n\n    self.write(results[component] if component else results)", "response": "Get the next available component"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nyielding the next available component from the cluster.", "response": "def get(self):\n    '''\n    :return:\n    '''\n    cluster = self.get_argument(\"cluster\")\n    environ = self.get_argument(\"environ\")\n    topology = self.get_argument(\"topology\")\n    component = self.get_argument(\"component\", default=None)\n    metric = self.get_argument(\"metric\")\n    instances = self.get_argument(\"instance\")\n    start = self.get_argument(\"starttime\")\n    end = self.get_argument(\"endtime\")\n    maxquery = self.get_argument(\"max\", default=False)\n    timerange = (start, end)\n    compnames = [component]\n\n    # fetch the metrics\n    futures = {}\n    if metric == \"backpressure\":\n      for comp in compnames:\n        future = query_handler.fetch_backpressure(cluster, metric, topology, component,\n                                                  instances, timerange, maxquery, environ)\n        futures[comp] = future\n    else:\n      fetch = query_handler.fetch_max if maxquery else query_handler.fetch\n      for comp in compnames:\n        future = fetch(cluster, metric, topology, component,\n                       instances, timerange, environ)\n        futures[comp] = future\n\n    results = yield futures\n    self.write(results[component] if component else results)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets method returns a list of objects", "response": "def get(self, path):\n    \"\"\" get method \"\"\"\n    if path is None:\n      return {}\n\n    if not utils.check_path(path):\n      self.write(\"Only relative paths are allowed\")\n      self.set_status(403)\n      self.finish()\n      return\n\n    offset = self.get_argument(\"offset\", default=-1)\n    length = self.get_argument(\"length\", default=-1)\n    if not os.path.isfile(path):\n      return {}\n    data = utils.read_chunk(path, offset=offset, length=length, escape_data=True)\n    self.write(json.dumps(data))\n    self.finish()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nimplement Pulsar Spout s initialize method", "response": "def initialize(self, config, context):\n    \"\"\"Implements Pulsar Spout's initialize method\"\"\"\n    self.logger.info(\"Initializing PulsarSpout with the following\")\n    self.logger.info(\"Component-specific config: \\n%s\" % str(config))\n    self.logger.info(\"Context: \\n%s\" % str(context))\n\n    self.emit_count = 0\n    self.ack_count = 0\n    self.fail_count = 0\n\n    if not PulsarSpout.serviceUrl in config or not PulsarSpout.topicName in config:\n      self.logger.fatal(\"Need to specify both serviceUrl and topicName\")\n    self.pulsar_cluster = str(config[PulsarSpout.serviceUrl])\n    self.topic = str(config[PulsarSpout.topicName])\n    mode = config[api_constants.TOPOLOGY_RELIABILITY_MODE]\n    if mode == api_constants.TopologyReliabilityMode.ATLEAST_ONCE:\n      self.acking_timeout = 1000 * int(config[api_constants.TOPOLOGY_MESSAGE_TIMEOUT_SECS])\n    else:\n      self.acking_timeout = 30000\n    if PulsarSpout.receiveTimeoutMs in config:\n      self.receive_timeout_ms = config[PulsarSpout.receiveTimeoutMs]\n    else:\n      self.receive_timeout_ms = 10\n    if PulsarSpout.deserializer in config:\n      self.deserializer = config[PulsarSpout.deserializer]\n      if not callable(self.deserializer):\n        self.logger.fatal(\"Pulsar Message Deserializer needs to be callable\")\n    else:\n      self.deserializer = self.default_deserializer\n\n    # First generate the config\n    self.logConfFileName = GenerateLogConfig(context)\n    self.logger.info(\"Generated LogConf at %s\" % self.logConfFileName)\n\n    # We currently use the high level consumer API\n    # For supporting effectively once, we will need to switch\n    # to using lower level Reader API, when it becomes\n    # available in python\n    self.client = pulsar.Client(self.pulsar_cluster, log_conf_file_path=self.logConfFileName)\n    self.logger.info(\"Setup Client with cluster %s\" % self.pulsar_cluster)\n    try:\n      self.consumer = self.client.subscribe(self.topic, context.get_topology_name(),\n                                            consumer_type=pulsar.ConsumerType.Failover,\n                                            unacked_messages_timeout_ms=self.acking_timeout)\n    except Exception as e:\n      self.logger.fatal(\"Pulsar client subscription failed: %s\" % str(e))\n\n    self.logger.info(\"Subscribed to topic %s\" % self.topic)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting all the values for the cluster and environ.", "response": "def get(self):\n    \"\"\" get method \"\"\"\n    # Get all the values for parameter \"cluster\".\n    clusters = self.get_arguments(constants.PARAM_CLUSTER)\n\n    # Get all the values for parameter \"environ\".\n    environs = self.get_arguments(constants.PARAM_ENVIRON)\n\n    role = self.get_argument_role()\n\n    ret = {}\n    topologies = self.tracker.topologies\n    for topology in topologies:\n      cluster = topology.cluster\n      environ = topology.environ\n      if not cluster or not environ:\n        continue\n\n      # This cluster is not asked for.\n      # Note that \"if not clusters\", then\n      # we show for all the clusters.\n      if clusters and cluster not in clusters:\n        continue\n\n      # This environ is not asked for.\n      # Note that \"if not environs\", then\n      # we show for all the environs.\n      if environs and environ not in environs:\n        continue\n\n      if cluster not in ret:\n        ret[cluster] = {}\n      if environ not in ret[cluster]:\n        ret[cluster][environ] = {}\n      try:\n        topology_info = self.tracker.getTopologyInfo(topology.name, cluster, role, environ)\n        if topology_info and \"execution_state\" in topology_info:\n          ret[cluster][environ][topology.name] = topology_info[\"execution_state\"]\n      except Exception:\n        # Do nothing\n        pass\n    self.write_success_response(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfetches Instance jstack from heron - shell.", "response": "def getInstanceJstack(self, topology_info, instance_id):\n    \"\"\"\n    Fetches Instance jstack from heron-shell.\n    \"\"\"\n    pid_response = yield getInstancePid(topology_info, instance_id)\n    try:\n      http_client = tornado.httpclient.AsyncHTTPClient()\n      pid_json = json.loads(pid_response)\n      pid = pid_json['stdout'].strip()\n      if pid == '':\n        raise Exception('Failed to get pid')\n      endpoint = utils.make_shell_endpoint(topology_info, instance_id)\n      url = \"%s/jstack/%s\" % (endpoint, pid)\n      response = yield http_client.fetch(url)\n      Log.debug(\"HTTP call for url: %s\", url)\n      raise tornado.gen.Return(response.body)\n    except tornado.httpclient.HTTPError as e:\n      raise Exception(str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating the parser for the update command", "response": "def create_parser(subparsers):\n  \"\"\" Create the parse for the update command \"\"\"\n  parser = subparsers.add_parser(\n      'update',\n      help='Update a topology',\n      usage=\"%(prog)s [options] cluster/[role]/[env] <topology-name> \"\n      + \"[--component-parallelism <name:value>] \"\n      + \"[--container-number value] \"\n      + \"[--runtime-config [component:]<name:value>]\",\n      add_help=True)\n\n  args.add_titles(parser)\n  args.add_cluster_role_env(parser)\n  args.add_topology(parser)\n\n  args.add_config(parser)\n  args.add_dry_run(parser)\n  args.add_service_url(parser)\n  args.add_verbose(parser)\n\n  # Special parameters for update command\n  def parallelism_type(value):\n    pattern = re.compile(r\"^[\\w\\.-]+:[\\d]+$\")\n    if not pattern.match(value):\n      raise argparse.ArgumentTypeError(\n          \"Invalid syntax for component parallelism (<component_name:value>): %s\" % value)\n    return value\n\n  parser.add_argument(\n      '--component-parallelism',\n      action='append',\n      type=parallelism_type,\n      required=False,\n      help='Component name and the new parallelism value '\n      + 'colon-delimited: <component_name>:<parallelism>')\n\n  def runtime_config_type(value):\n    pattern = re.compile(r\"^([\\w\\.-]+:){1,2}[\\w\\.-]+$\")\n    if not pattern.match(value):\n      raise argparse.ArgumentTypeError(\n          \"Invalid syntax for runtime config ([component:]<name:value>): %s\"\n          % value)\n    return value\n\n  parser.add_argument(\n      '--runtime-config',\n      action='append',\n      type=runtime_config_type,\n      required=False,\n      help='Runtime configurations for topology and components '\n      + 'colon-delimited: [component:]<name>:<value>')\n\n  def container_number_type(value):\n    pattern = re.compile(r\"^\\d+$\")\n    if not pattern.match(value):\n      raise argparse.ArgumentTypeError(\n          \"Invalid syntax for container number (value): %s\"\n          % value)\n    return value\n\n  parser.add_argument(\n      '--container-number',\n      action='append',\n      type=container_number_type,\n      required=False,\n      help='Number of containers <value>')\n\n  parser.set_defaults(subcommand='update')\n  return parser"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef build_extra_args_dict(cl_args):\n  # Check parameters\n  component_parallelism = cl_args['component_parallelism']\n  runtime_configs = cl_args['runtime_config']\n  container_number = cl_args['container_number']\n  # Users need to provide either (component-parallelism || container_number) or runtime-config\n  if (component_parallelism and runtime_configs) or (container_number and runtime_configs):\n    raise Exception(\n        \"(component-parallelism or container_num) and runtime-config \" +\n        \"can't be updated at the same time\")\n\n  dict_extra_args = {}\n\n  nothing_set = True\n  if component_parallelism:\n    dict_extra_args.update({'component_parallelism': component_parallelism})\n    nothing_set = False\n\n  if container_number:\n    dict_extra_args.update({'container_number': container_number})\n    nothing_set = False\n\n  if runtime_configs:\n    dict_extra_args.update({'runtime_config': runtime_configs})\n    nothing_set = False\n\n  if nothing_set:\n    raise Exception(\n        \"Missing arguments --component-parallelism or --runtime-config or --container-number\")\n\n  if cl_args['dry_run']:\n    dict_extra_args.update({'dry_run': True})\n    if 'dry_run_format' in cl_args:\n      dict_extra_args.update({'dry_run_format': cl_args[\"dry_run_format\"]})\n\n  return dict_extra_args", "response": "Build a dictionary of extra arguments for the current component."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef convert_args_dict_to_list(dict_extra_args):\n  list_extra_args = []\n  if 'component_parallelism' in dict_extra_args:\n    list_extra_args += [\"--component_parallelism\",\n                        ','.join(dict_extra_args['component_parallelism'])]\n  if 'runtime_config' in dict_extra_args:\n    list_extra_args += [\"--runtime_config\",\n                        ','.join(dict_extra_args['runtime_config'])]\n  if 'container_number' in dict_extra_args:\n    list_extra_args += [\"--container_number\",\n                        ','.join(dict_extra_args['container_number'])]\n  if 'dry_run' in dict_extra_args and dict_extra_args['dry_run']:\n    list_extra_args += ['--dry_run']\n  if 'dry_run_format' in dict_extra_args:\n    list_extra_args += ['--dry_run_format', dict_extra_args['dry_run_format']]\n\n  return list_extra_args", "response": "converts dict_extra_args to list of extra args"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrun the update command", "response": "def run(command, parser, cl_args, unknown_args):\n  \"\"\" run the update command \"\"\"\n\n  Log.debug(\"Update Args: %s\", cl_args)\n\n  # Build jar list\n  extra_lib_jars = jars.packing_jars()\n  action = \"update topology%s\" % (' in dry-run mode' if cl_args[\"dry_run\"] else '')\n\n  # Build extra args\n  dict_extra_args = {}\n  try:\n    dict_extra_args = build_extra_args_dict(cl_args)\n  except Exception as err:\n    return SimpleResult(Status.InvocationError, err.message)\n\n  # Execute\n  if cl_args['deploy_mode'] == config.SERVER_MODE:\n    return cli_helper.run_server(command, cl_args, action, dict_extra_args)\n  else:\n    # Convert extra argument to commandline format and then execute\n    list_extra_args = convert_args_dict_to_list(dict_extra_args)\n    return cli_helper.run_direct(command, cl_args, action, list_extra_args, extra_lib_jars)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef getInstancePid(topology_info, instance_id):\n  try:\n    http_client = tornado.httpclient.AsyncHTTPClient()\n    endpoint = utils.make_shell_endpoint(topology_info, instance_id)\n    url = \"%s/pid/%s\" % (endpoint, instance_id)\n    Log.debug(\"HTTP call for url: %s\", url)\n    response = yield http_client.fetch(url)\n    raise tornado.gen.Return(response.body)\n  except tornado.httpclient.HTTPError as e:\n    raise Exception(str(e))", "response": "Retrieves Instance pid from heron - shell."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets method returns a generator that yields the instance PID of the current instance", "response": "def get(self):\n    \"\"\" get method \"\"\"\n    try:\n      cluster = self.get_argument_cluster()\n      role = self.get_argument_role()\n      environ = self.get_argument_environ()\n      topology_name = self.get_argument_topology()\n      instance = self.get_argument_instance()\n      topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)\n      result = yield getInstancePid(topology_info, instance)\n      self.write_success_response(result)\n    except Exception as e:\n      Log.debug(traceback.format_exc())\n      self.write_error_response(e)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck if a given gtype is sane", "response": "def is_grouping_sane(cls, gtype):\n    \"\"\"Checks if a given gtype is sane\"\"\"\n    if gtype == cls.SHUFFLE or gtype == cls.ALL or gtype == cls.LOWEST or gtype == cls.NONE:\n      return True\n    elif isinstance(gtype, cls.FIELDS):\n      return gtype.gtype == topology_pb2.Grouping.Value(\"FIELDS\") and \\\n             gtype.fields is not None\n    elif isinstance(gtype, cls.CUSTOM):\n      return gtype.gtype == topology_pb2.Grouping.Value(\"CUSTOM\") and \\\n             gtype.python_serialized is not None\n    else:\n      #pylint: disable=fixme\n      #TODO: DIRECT are not supported yet\n      return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef fields(cls, *fields):\n    if len(fields) == 1 and isinstance(fields[0], list):\n      fields = fields[0]\n    else:\n      fields = list(fields)\n\n    for i in fields:\n      if not isinstance(i, str):\n        raise TypeError(\"Non-string cannot be specified in fields\")\n\n    if not fields:\n      raise ValueError(\"List cannot be empty for fields grouping\")\n\n    return cls.FIELDS(gtype=topology_pb2.Grouping.Value(\"FIELDS\"),\n                      fields=fields)", "response": "Create a new FIELDS object with the given list of fields."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef custom_serialized(cls, serialized, is_java=True):\n    if not isinstance(serialized, bytes):\n      raise TypeError(\"Argument to custom_serialized() must be \"\n                      \"a serialized Python class as bytes, given: %s\" % str(serialized))\n    if not is_java:\n      return cls.CUSTOM(gtype=topology_pb2.Grouping.Value(\"CUSTOM\"),\n                        python_serialized=serialized)\n    else:\n      raise NotImplementedError(\"Custom grouping implemented in Java for Python topology\"\n                                \"is not yet supported.\")", "response": "Custom grouping from a given serialized string"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef register_metrics(self, metrics_collector, interval):\n    for field, metrics in self.metrics.items():\n      metrics_collector.register_metric(field, metrics, interval)", "response": "Registers its metrics to a given metrics collector with a given interval"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update_count(self, name, incr_by=1, key=None):\n    if name not in self.metrics:\n      Log.error(\"In update_count(): %s is not registered in the metric\", name)\n\n    if key is None and isinstance(self.metrics[name], CountMetric):\n      self.metrics[name].incr(incr_by)\n    elif key is not None and isinstance(self.metrics[name], MultiCountMetric):\n      self.metrics[name].incr(key, incr_by)\n    else:\n      Log.error(\"In update_count(): %s is registered but not supported with this method\", name)", "response": "Updates the value of a count metric or MultiCountMetric."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nupdate the value of a reduced metric or MultiReducedMetric.", "response": "def update_reduced_metric(self, name, value, key=None):\n    \"\"\"Update the value of ReducedMetric or MultiReducedMetric\n\n    :type name: str\n    :param name: name of the registered metric to be updated.\n    :param value: specifies a value to be reduced.\n    :type key: str or None\n    :param key: specifies a key for MultiReducedMetric. Needs to be `None` for updating\n                ReducedMetric.\n    \"\"\"\n    if name not in self.metrics:\n      Log.error(\"In update_reduced_metric(): %s is not registered in the metric\", name)\n\n    if key is None and isinstance(self.metrics[name], ReducedMetric):\n      self.metrics[name].update(value)\n    elif key is not None and isinstance(self.metrics[name], MultiReducedMetric):\n      self.metrics[name].update(key, value)\n    else:\n      Log.error(\"In update_count(): %s is registered but not supported with this method\", name)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nupdate the received packet count and size metrics.", "response": "def update_received_packet(self, received_pkt_size_bytes):\n    \"\"\"Update received packet metrics\"\"\"\n    self.update_count(self.RECEIVED_PKT_COUNT)\n    self.update_count(self.RECEIVED_PKT_SIZE, incr_by=received_pkt_size_bytes)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update_sent_packet(self, sent_pkt_size_bytes):\n    self.update_count(self.SENT_PKT_COUNT)\n    self.update_count(self.SENT_PKT_SIZE, incr_by=sent_pkt_size_bytes)", "response": "Update sent packet metrics"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef register_metrics(self, context):\n    sys_config = system_config.get_sys_config()\n    interval = float(sys_config[constants.HERON_METRICS_EXPORT_INTERVAL_SEC])\n    collector = context.get_metrics_collector()\n    super(ComponentMetrics, self).register_metrics(collector, interval)", "response": "Registers metrics to context\n   "}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef serialize_data_tuple(self, stream_id, latency_in_ns):\n    self.update_count(self.TUPLE_SERIALIZATION_TIME_NS, incr_by=latency_in_ns, key=stream_id)", "response": "Apply update to serialization metrics"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _init_multi_count_metrics(self, pplan_helper):\n    to_init = [self.metrics[i] for i in self.to_multi_init\n               if i in self.metrics and isinstance(self.metrics[i], MultiCountMetric)]\n    for out_stream in pplan_helper.get_my_spout().outputs:\n      stream_id = out_stream.stream.id\n      for metric in to_init:\n        metric.add_key(stream_id)", "response": "Initializes the default values for a necessary set of MultiCountMetrics"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef next_tuple(self, latency_in_ns):\n    self.update_reduced_metric(self.NEXT_TUPLE_LATENCY, latency_in_ns)\n    self.update_count(self.NEXT_TUPLE_COUNT)", "response": "Apply updates to the next tuple metric"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\napply updates to the ack metrics", "response": "def acked_tuple(self, stream_id, complete_latency_ns):\n    \"\"\"Apply updates to the ack metrics\"\"\"\n    self.update_count(self.ACK_COUNT, key=stream_id)\n    self.update_reduced_metric(self.COMPLETE_LATENCY, complete_latency_ns, key=stream_id)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\napply updates to the fail metrics for a given stream", "response": "def failed_tuple(self, stream_id, fail_latency_ns):\n    \"\"\"Apply updates to the fail metrics\"\"\"\n    self.update_count(self.FAIL_COUNT, key=stream_id)\n    self.update_reduced_metric(self.FAIL_LATENCY, fail_latency_ns, key=stream_id)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninitialize the default values for a necessary set of MultiCountMetrics", "response": "def _init_multi_count_metrics(self, pplan_helper):\n    \"\"\"Initializes the default values for a necessary set of MultiCountMetrics\"\"\"\n    # inputs\n    to_in_init = [self.metrics[i] for i in self.inputs_init\n                  if i in self.metrics and isinstance(self.metrics[i], MultiCountMetric)]\n    for in_stream in pplan_helper.get_my_bolt().inputs:\n      stream_id = in_stream.stream.id\n      global_stream_id = in_stream.stream.component_name + \"/\" + stream_id\n      for metric in to_in_init:\n        metric.add_key(stream_id)\n        metric.add_key(global_stream_id)\n    # outputs\n    to_out_init = [self.metrics[i] for i in self.outputs_init\n                   if i in self.metrics and isinstance(self.metrics[i], MultiCountMetric)]\n    for out_stream in pplan_helper.get_my_bolt().outputs:\n      stream_id = out_stream.stream.id\n      for metric in to_out_init:\n        metric.add_key(stream_id)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\napplies updates to the execute metrics", "response": "def execute_tuple(self, stream_id, source_component, latency_in_ns):\n    \"\"\"Apply updates to the execute metrics\"\"\"\n    self.update_count(self.EXEC_COUNT, key=stream_id)\n    self.update_reduced_metric(self.EXEC_LATENCY, latency_in_ns, stream_id)\n    self.update_count(self.EXEC_TIME_NS, incr_by=latency_in_ns, key=stream_id)\n\n    global_stream_id = source_component + \"/\" + stream_id\n    self.update_count(self.EXEC_COUNT, key=global_stream_id)\n    self.update_reduced_metric(self.EXEC_LATENCY, latency_in_ns, global_stream_id)\n    self.update_count(self.EXEC_TIME_NS, incr_by=latency_in_ns, key=global_stream_id)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\napply updates to the deserialization metrics", "response": "def deserialize_data_tuple(self, stream_id, source_component, latency_in_ns):\n    \"\"\"Apply updates to the deserialization metrics\"\"\"\n    self.update_count(self.TUPLE_DESERIALIZATION_TIME_NS, incr_by=latency_in_ns, key=stream_id)\n    global_stream_id = source_component + \"/\" + stream_id\n    self.update_count(self.TUPLE_DESERIALIZATION_TIME_NS, incr_by=latency_in_ns,\n                      key=global_stream_id)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\napplies updates to the ack metrics", "response": "def acked_tuple(self, stream_id, source_component, latency_in_ns):\n    \"\"\"Apply updates to the ack metrics\"\"\"\n    self.update_count(self.ACK_COUNT, key=stream_id)\n    self.update_reduced_metric(self.PROCESS_LATENCY, latency_in_ns, stream_id)\n    global_stream_id = source_component + '/' + stream_id\n    self.update_count(self.ACK_COUNT, key=global_stream_id)\n    self.update_reduced_metric(self.PROCESS_LATENCY, latency_in_ns, global_stream_id)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef failed_tuple(self, stream_id, source_component, latency_in_ns):\n    self.update_count(self.FAIL_COUNT, key=stream_id)\n    self.update_reduced_metric(self.FAIL_LATENCY, latency_in_ns, stream_id)\n    global_stream_id = source_component + '/' + stream_id\n    self.update_count(self.FAIL_COUNT, key=global_stream_id)\n    self.update_reduced_metric(self.FAIL_LATENCY, latency_in_ns, global_stream_id)", "response": "Apply updates to the fail metrics"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef register_metric(self, name, metric, time_bucket_in_sec):\n    if name in self.metrics_map:\n      raise RuntimeError(\"Another metric has already been registered with name: %s\" % name)\n\n    Log.debug(\"Register metric: %s, with interval: %s\", name, str(time_bucket_in_sec))\n    self.metrics_map[name] = metric\n\n    if time_bucket_in_sec in self.time_bucket_in_sec_to_metrics_name:\n      self.time_bucket_in_sec_to_metrics_name[time_bucket_in_sec].append(name)\n    else:\n      self.time_bucket_in_sec_to_metrics_name[time_bucket_in_sec] = [name]\n      self._register_timer_task(time_bucket_in_sec)", "response": "Registers a metric with the metrics manager"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef poll(self):\n    try:\n      # non-blocking\n      ret = self._buffer.get(block=False)\n      if self._producer_callback is not None:\n        self._producer_callback()\n      return ret\n    except Queue.Empty:\n      Log.debug(\"%s: Empty in poll()\" % str(self))\n      raise Queue.Empty", "response": "Poll from the buffer\n   "}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\noffering to the buffer and return True if the buffer is full.", "response": "def offer(self, item):\n    \"\"\"Offer to the buffer\n\n    It is a non-blocking operation, and when the buffer is full, it raises Queue.Full exception\n    \"\"\"\n    try:\n      # non-blocking\n      self._buffer.put(item, block=False)\n      if self._consumer_callback is not None:\n        self._consumer_callback()\n      return True\n    except Queue.Full:\n      Log.debug(\"%s: Full in offer()\" % str(self))\n      raise Queue.Full"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef parse(version):\n    match = _REGEX.match(version)\n    if match is None:\n        raise ValueError('%s is not valid SemVer string' % version)\n\n    verinfo = match.groupdict()\n\n    verinfo['major'] = int(verinfo['major'])\n    verinfo['minor'] = int(verinfo['minor'])\n    verinfo['patch'] = int(verinfo['patch'])\n\n    return verinfo", "response": "Parse a SemVer string into major minor patch pre - release build parts."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a parser for the command.", "response": "def create_parser(subparsers, action, help_arg):\n  '''\n  :param subparsers:\n  :param action:\n  :param help_arg:\n  :return:\n  '''\n  parser = subparsers.add_parser(\n      action,\n      help=help_arg,\n      usage=\"%(prog)s [options] cluster/[role]/[env] <topology-name>\",\n      add_help=True)\n\n  args.add_titles(parser)\n  args.add_cluster_role_env(parser)\n  args.add_topology(parser)\n\n  args.add_config(parser)\n  args.add_service_url(parser)\n  args.add_verbose(parser)\n\n  parser.set_defaults(subcommand=action)\n  return parser"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run_server(command, cl_args, action, extra_args=dict()):\n  '''\n  helper function to take action on topologies using REST API\n  :param command:\n  :param cl_args:\n  :param action:        description of action taken\n  :return:\n  '''\n  topology_name = cl_args['topology-name']\n\n  service_endpoint = cl_args['service_url']\n  apiroute = rest.ROUTE_SIGNATURES[command][1] % (\n      cl_args['cluster'],\n      cl_args['role'],\n      cl_args['environ'],\n      topology_name\n  )\n  service_apiurl = service_endpoint + apiroute\n  service_method = rest.ROUTE_SIGNATURES[command][0]\n\n  # convert the dictionary to a list of tuples\n  data = flatten_args(extra_args)\n\n  err_msg = \"Failed to %s: %s\" % (action, topology_name)\n  succ_msg = \"Successfully %s: %s\" % (action, topology_name)\n\n  try:\n    r = service_method(service_apiurl, data=data)\n    s = Status.Ok if r.status_code == requests.codes.ok else Status.HeronError\n    if r.status_code != requests.codes.ok:\n      Log.error(r.json().get('message', \"Unknown error from API server %d\" % r.status_code))\n  except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:\n    Log.error(err)\n    return SimpleResult(Status.HeronError, err_msg, succ_msg)\n\n  return SimpleResult(s, err_msg, succ_msg)", "response": "helper function to take action on topologies using REST API\n athorian"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef run_direct(command, cl_args, action, extra_args=[], extra_lib_jars=[]):\n  '''\n  helper function to take action on topologies\n  :param command:\n  :param cl_args:\n  :param action:        description of action taken\n  :return:\n  '''\n  topology_name = cl_args['topology-name']\n\n  new_args = [\n      \"--cluster\", cl_args['cluster'],\n      \"--role\", cl_args['role'],\n      \"--environment\", cl_args['environ'],\n      \"--submit_user\", cl_args['submit_user'],\n      \"--heron_home\", config.get_heron_dir(),\n      \"--config_path\", cl_args['config_path'],\n      \"--override_config_file\", cl_args['override_config_file'],\n      \"--release_file\", config.get_heron_release_file(),\n      \"--topology_name\", topology_name,\n      \"--command\", command,\n  ]\n  new_args += extra_args\n  lib_jars = config.get_heron_libs(jars.scheduler_jars() + jars.statemgr_jars())\n  lib_jars += extra_lib_jars\n\n  if Log.getEffectiveLevel() == logging.DEBUG:\n    new_args.append(\"--verbose\")\n\n  # invoke the runtime manager to kill the topology\n  result = execute.heron_class(\n      'org.apache.heron.scheduler.RuntimeManagerMain',\n      lib_jars,\n      extra_jars=[],\n      args=new_args\n  )\n\n  err_msg = \"Failed to %s: %s\" % (action, topology_name)\n  succ_msg = \"Successfully %s: %s\" % (action, topology_name)\n  result.add_context(err_msg, succ_msg)\n  return result", "response": "helper function to take action on topologies\n helper function to take action on topologies\n  helper function to take action on topologies\n \u00abdirect\u00bb"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_all_zk_state_managers(conf):\n  state_managers = []\n  state_locations = conf.get_state_locations_of_type(\"zookeeper\")\n  for location in state_locations:\n    name = location['name']\n    hostport = location['hostport']\n    hostportlist = []\n    for hostportpair in hostport.split(','):\n      host = None\n      port = None\n      if ':' in hostport:\n        hostandport = hostportpair.split(':')\n        if len(hostandport) == 2:\n          host = hostandport[0]\n          port = int(hostandport[1])\n      if not host or not port:\n        raise Exception(\"Hostport for %s must be of the format 'host:port'.\" % (name))\n      hostportlist.append((host, port))\n    tunnelhost = location['tunnelhost']\n    rootpath = location['rootpath']\n    LOG.info(\"Connecting to zk hostports: \" + str(hostportlist) + \" rootpath: \" + rootpath)\n    state_manager = ZkStateManager(name, hostportlist, rootpath, tunnelhost)\n    state_managers.append(state_manager)\n\n  return state_managers", "response": "Creates all the zookeeper state_managers and returns\n athe"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns all the file state managers.", "response": "def get_all_file_state_managers(conf):\n  \"\"\"\n  Returns all the file state_managers.\n  \"\"\"\n  state_managers = []\n  state_locations = conf.get_state_locations_of_type(\"file\")\n  for location in state_locations:\n    name = location['name']\n    rootpath = os.path.expanduser(location['rootpath'])\n    LOG.info(\"Connecting to file state with rootpath: \" + rootpath)\n    state_manager = FileStateManager(name, rootpath)\n    state_managers.append(state_manager)\n\n  return state_managers"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nincrements the value of a given key by to_add.", "response": "def incr(self, key, to_add=1):\n    \"\"\"Increments the value of a given key by ``to_add``\"\"\"\n    if key not in self.value:\n      self.value[key] = CountMetric()\n    self.value[key].incr(to_add)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef update(self, key, value):\n    if key not in self.value:\n      self.value[key] = ReducedMetric(self.reducer)\n\n    self.value[key].update(value)", "response": "Updates a value of a given key and apply reduction"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding a new key to this metric", "response": "def add_key(self, key):\n    \"\"\"Adds a new key to this metric\"\"\"\n    if key not in self.value:\n      self.value[key] = ReducedMetric(self.reducer)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef add_data_tuple(self, stream_id, new_data_tuple, tuple_size_in_bytes):\n    if (self.current_data_tuple_set is None) or \\\n        (self.current_data_tuple_set.stream.id != stream_id) or \\\n        (len(self.current_data_tuple_set.tuples) >= self.data_tuple_set_capacity) or \\\n        (self.current_data_tuple_size_in_bytes >= self.max_data_tuple_size_in_bytes):\n      self._init_new_data_tuple(stream_id)\n\n    added_tuple = self.current_data_tuple_set.tuples.add()\n    added_tuple.CopyFrom(new_data_tuple)\n\n    self.current_data_tuple_size_in_bytes += tuple_size_in_bytes\n    self.total_data_emitted_in_bytes += tuple_size_in_bytes", "response": "Add a new data tuple to the currently buffered set of tuples"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef add_control_tuple(self, new_control_tuple, tuple_size_in_bytes, is_ack):\n    if self.current_control_tuple_set is None:\n      self._init_new_control_tuple()\n    elif is_ack and (len(self.current_control_tuple_set.fails) > 0 or\n                     len(self.current_control_tuple_set.acks) >= self.control_tuple_set_capacity):\n      self._init_new_control_tuple()\n    elif not is_ack and \\\n        (len(self.current_control_tuple_set.acks) > 0 or\n         len(self.current_control_tuple_set.fails) >= self.control_tuple_set_capacity):\n      self._init_new_control_tuple()\n\n    if is_ack:\n      added_tuple = self.current_control_tuple_set.acks.add()\n    else:\n      added_tuple = self.current_control_tuple_set.fails.add()\n\n    added_tuple.CopyFrom(new_control_tuple)\n\n    self.total_data_emitted_in_bytes += tuple_size_in_bytes", "response": "Adds a new control tuple to the currently buffered set of tuples\n   "}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd the checkpoint state message to be sent back the stmgr", "response": "def add_ckpt_state(self, ckpt_id, ckpt_state):\n    \"\"\"Add the checkpoint state message to be sent back the stmgr\n\n    :param ckpt_id: The id of the checkpoint\n    :ckpt_state: The checkpoint state\n    \"\"\"\n    # first flush any buffered tuples\n    self._flush_remaining()\n    msg = ckptmgr_pb2.StoreInstanceStateCheckpoint()\n    istate = ckptmgr_pb2.InstanceStateCheckpoint()\n    istate.checkpoint_id = ckpt_id\n    istate.state = ckpt_state\n    msg.state.CopyFrom(istate)\n    self._push_tuple_to_stream(msg)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef create_parser(subparsers):\n  '''\n  :param subparsers:\n  :return:\n  '''\n  parser = subparsers.add_parser(\n      'config',\n      help='Config properties for a cluster',\n      usage=\"%(prog)s [cluster]\",\n      add_help=True)\n\n  parser.add_argument(\n      'cluster',\n      help='Cluster to configure'\n  )\n\n  ex_subparsers = parser.add_subparsers(\n      title=\"Commands\",\n      description=None)\n\n  # add config list parser\n  list_parser = ex_subparsers.add_parser(\n      'list',\n      help='List config properties for a cluster',\n      usage=\"%(prog)s\",\n      add_help=True)\n  list_parser.set_defaults(configcommand='list')\n\n  # add config set parser\n  set_parser = ex_subparsers.add_parser(\n      'set',\n      help='Set a cluster config property',\n      usage=\"%(prog)s [property] [value]\",\n      add_help=True)\n\n  set_parser.add_argument(\n      'property',\n      help='Config property to set'\n  )\n\n  set_parser.add_argument(\n      'value',\n      help='Value of config property'\n  )\n  set_parser.set_defaults(configcommand='set')\n\n  # add config unset parser\n  unset_parser = ex_subparsers.add_parser(\n      'unset',\n      help='Unset a cluster config property',\n      usage=\"%(prog)s [property]\",\n      add_help=True)\n\n  unset_parser.add_argument(\n      'property',\n      help='Config property to unset'\n  )\n  unset_parser.set_defaults(configcommand='unset')\n\n  parser.set_defaults(subcommand='config')\n  return parser", "response": "Create a parser for the cluster config command"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrun command with no arguments", "response": "def run(command, parser, cl_args, unknown_args):\n  '''\n  :param command:\n  :param parser:\n  :param args:\n  :param unknown_args:\n  :return:\n  '''\n  configcommand = cl_args.get('configcommand', None)\n  if configcommand == 'set':\n    return _set(cl_args)\n  elif configcommand == 'unset':\n    return _unset(cl_args)\n  else:\n    return _list(cl_args)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncheck whether the java classpath contains valid path entries.", "response": "def valid_java_classpath(classpath):\n  '''\n  Given a java classpath, check whether the path entries are valid or not\n  '''\n  paths = classpath.split(':')\n  for path_entry in paths:\n    if not valid_path(path_entry.strip()):\n      return False\n  return True"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nadd an edge between two nodes.", "response": "def add_edge(self, U, V):\n    '''\n    :param U:\n    :param V:\n    :return:\n    '''\n    if not U in self.edges:\n      self.edges[U] = set()\n    if not V in self.edges:\n      self.edges[V] = set()\n    if not V in self.edges[U]:\n      self.edges[U].add(V)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the maximum distance between any vertex and U in the connected component containing U.", "response": "def bfs_depth(self, U):\n    '''\n    Returns the maximum distance between any vertex and U in the connected\n    component containing U\n    :param U:\n    :return:\n    '''\n    bfs_queue = [[U, 0]]  # Stores the vertices whose BFS hadn't been completed.\n    visited = set()\n    max_depth = 0\n    while bfs_queue:\n      [V, depth] = bfs_queue.pop()\n      if max_depth < depth:\n        max_depth = depth\n      visited.add(V)\n      adj_set = self.edges[V]\n      for W in adj_set:\n        if W not in visited:\n          bfs_queue.append([W, depth + 1])\n    return max_depth"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the maximum distance between any vertex and U in the connected component containing U.", "response": "def diameter(self):\n    '''\n    Returns the maximum distance between any vertex and U in the connected\n    component containing U\n    :return:\n    '''\n    diameter = 0\n    for U in self.edges:\n      depth = self.bfs_depth(U)\n      if depth > diameter:\n        diameter = depth\n    return diameter"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _get_deps_list(abs_path_to_pex):\n  pex = zipfile.ZipFile(abs_path_to_pex, mode='r')\n  deps = list(set([re.match(egg_regex, i).group(1) for i in pex.namelist()\n                   if re.match(egg_regex, i) is not None]))\n  return deps", "response": "Get a list of paths to included dependencies in the specified pex file\n\n Get a list of paths to included dependencies in the specified pex file\n\n"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef load_pex(path_to_pex, include_deps=True):\n  abs_path_to_pex = os.path.abspath(path_to_pex)\n  Log.debug(\"Add a pex to the path: %s\" % abs_path_to_pex)\n  if abs_path_to_pex not in sys.path:\n    sys.path.insert(0, os.path.dirname(abs_path_to_pex))\n\n  # add dependencies to path\n  if include_deps:\n    for dep in _get_deps_list(abs_path_to_pex):\n      to_join = os.path.join(os.path.dirname(abs_path_to_pex), dep)\n      if to_join not in sys.path:\n        Log.debug(\"Add a new dependency to the path: %s\" % dep)\n        sys.path.insert(0, to_join)\n\n  Log.debug(\"Python path: %s\" % str(sys.path))", "response": "Loads a pex file and its dependencies to the current python path"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef resolve_heron_suffix_issue(abs_pex_path, class_path):\n  # import top-level package named `heron` of a given pex file\n  importer = zipimport.zipimporter(abs_pex_path)\n  importer.load_module(\"heron\")\n\n  # remove 'heron' and the classname\n  to_load_lst = class_path.split('.')[1:-1]\n  loaded = ['heron']\n  loaded_mod = None\n  for to_load in to_load_lst:\n    sub_importer = zipimport.zipimporter(os.path.join(abs_pex_path, '/'.join(loaded)))\n    loaded_mod = sub_importer.load_module(to_load)\n    loaded.append(to_load)\n\n  return loaded_mod", "response": "Resolves duplicate package suffix problems with Heron instance pex file and class path."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nimports and loads a class from a given pex file path and python class name.", "response": "def import_and_get_class(path_to_pex, python_class_name):\n  \"\"\"Imports and load a class from a given pex file path and python class name\n\n  For example, if you want to get a class called `Sample` in\n  /some-path/sample.pex/heron/examples/src/python/sample.py,\n  ``path_to_pex`` needs to be ``/some-path/sample.pex``, and\n  ``python_class_name`` needs to be ``heron.examples.src.python.sample.Sample``\n  \"\"\"\n  abs_path_to_pex = os.path.abspath(path_to_pex)\n\n  Log.debug(\"Add a pex to the path: %s\" % abs_path_to_pex)\n  Log.debug(\"In import_and_get_class with cls_name: %s\" % python_class_name)\n  split = python_class_name.split('.')\n  from_path = '.'.join(split[:-1])\n  import_name = python_class_name.split('.')[-1]\n\n  Log.debug(\"From path: %s, import name: %s\" % (from_path, import_name))\n\n  # Resolve duplicate package suffix problem (heron.), if the top level package name is heron\n  if python_class_name.startswith(\"heron.\"):\n    try:\n      mod = resolve_heron_suffix_issue(abs_path_to_pex, python_class_name)\n      return getattr(mod, import_name)\n    except:\n      Log.error(\"Could not resolve class %s with special handling\" % python_class_name)\n\n  mod = __import__(from_path, fromlist=[import_name], level=-1)\n  Log.debug(\"Imported module: %s\" % str(mod))\n  return getattr(mod, import_name)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef new_source(self, source):\n\n    source_streamlet = None\n    if callable(source):\n      source_streamlet = SupplierStreamlet(source)\n    elif isinstance(source, Generator):\n      source_streamlet = GeneratorStreamlet(source)\n    else:\n      raise RuntimeError(\"Builder's new source has to be either a Generator or a function\")\n\n    self._sources.append(source_streamlet)\n    return source_streamlet", "response": "Adds a new source to the computation DAG"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef build(self, bldr):\n    stage_names = sets.Set()\n    for source in self._sources:\n      source._build(bldr, stage_names)\n    for source in self._sources:\n      if not source._all_built():\n        raise RuntimeError(\"Topology cannot be fully built! Are all sources added?\")", "response": "Builds the topology and returns the builder"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_state_manager_locations(cluster, state_manager_config_file='heron-conf/statemgr.yaml',\n                                 overrides={}):\n  \"\"\" Reads configs to determine which state manager to use and converts them to state manager\n  locations. Handles a subset of config wildcard substitution supported in the substitute method in\n  org.apache.heron.spi.common.Misc.java\"\"\"\n  with open(state_manager_config_file, 'r') as stream:\n    config = yaml.load(stream)\n\n  home_dir = os.path.expanduser(\"~\")\n  wildcards = {\n      \"~\" : home_dir,\n      \"${HOME}\" : home_dir,\n      \"${CLUSTER}\" : cluster,\n  }\n  if os.getenv('JAVA_HOME'):\n    wildcards[\"${JAVA_HOME}\"] = os.getenv('JAVA_HOME')\n\n  config = __replace(config, wildcards, state_manager_config_file)\n\n  # merge with overrides\n  if overrides:\n    config.update(overrides)\n\n  # need to convert from the format in statemgr.yaml to the format that the python state managers\n  # takes. first, set reasonable defaults to local\n  state_manager_location = {\n      'type': 'file',\n      'name': 'local',\n      'tunnelhost': '127.0.0.1',\n      'rootpath': '~/.herondata/repository/state/local',\n  }\n\n  # then map the statemgr.yaml config keys to the python state manager location\n  key_mappings = {\n      'heron.statemgr.connection.string': 'hostport',\n      'heron.statemgr.tunnel.host': 'tunnelhost',\n      'heron.statemgr.root.path': 'rootpath',\n  }\n  for config_key in key_mappings:\n    if config_key in config:\n      state_manager_location[key_mappings[config_key]] = config[config_key]\n\n  state_manager_class = config['heron.class.state.manager']\n  if state_manager_class == 'org.apache.heron.statemgr.zookeeper.curator.CuratorStateManager':\n    state_manager_location['type'] = 'zookeeper'\n    state_manager_location['name'] = 'zk'\n\n  return [state_manager_location]", "response": "Reads the state manager configuration file and converts it to state manager locations."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __replace(config, wildcards, config_file):\n  for config_key in config:\n    config_value = config[config_key]\n    original_value = config_value\n    if isinstance(config_value, str):\n      for token in wildcards:\n        if wildcards[token]:\n          config_value = config_value.replace(token, wildcards[token])\n      found = re.findall(r'\\${[A-Z_]+}', config_value)\n      if found:\n        raise ValueError(\"%s=%s in file %s contains unsupported or unset wildcard tokens: %s\" %\n                         (config_key, original_value, config_file, \", \".join(found)))\n      config[config_key] = config_value\n  return config", "response": "Replace all the values in config with the values in wildcards"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a map of command names and handlers that are used by the command line tool.", "response": "def get_command_handlers():\n  '''\n  Create a map of command names and handlers\n  '''\n  return {\n      'activate': activate,\n      'config': hconfig,\n      'deactivate': deactivate,\n      'help': cli_help,\n      'kill': kill,\n      'restart': restart,\n      'submit': submit,\n      'update': update,\n      'version': version\n  }"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef create_parser(command_handlers):\n  '''\n  Main parser\n  :return:\n  '''\n  parser = argparse.ArgumentParser(\n      prog='heron',\n      epilog=HELP_EPILOG,\n      formatter_class=config.SubcommandHelpFormatter,\n      add_help=True)\n\n  subparsers = parser.add_subparsers(\n      title=\"Available commands\",\n      metavar='<command> <options>')\n\n  command_list = sorted(command_handlers.items())\n  for command in command_list:\n    command[1].create_parser(subparsers)\n\n  return parser", "response": "Create a parser for the availabe command list."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun the command .", "response": "def run(handlers, command, parser, command_args, unknown_args):\n  '''\n  Run the command\n  :param command:\n  :param parser:\n  :param command_args:\n  :param unknown_args:\n  :return:\n  '''\n\n  if command in handlers:\n    return handlers[command].run(command, parser, command_args, unknown_args)\n  else:\n    err_context = 'Unknown subcommand: %s' % command\n    return result.SimpleResult(result.Status.InvocationError, err_context)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nremove all the files in the list of files", "response": "def cleanup(files):\n  '''\n  :param files:\n  :return:\n  '''\n  for cur_file in files:\n    if os.path.isdir(cur_file):\n      shutil.rmtree(cur_file)\n    else:\n      shutil.rmtree(os.path.dirname(cur_file))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchecking the server deployment mode for the given cluster", "response": "def server_deployment_mode(command, parser, cluster, cl_args):\n  '''\n  check the server deployment mode for the given cluster\n  if it is valid return the valid set of args\n  :param cluster:\n  :param cl_args:\n  :return:\n  '''\n  # Read the cluster definition, if not found\n  client_confs = cdefs.read_server_mode_cluster_definition(cluster, cl_args)\n\n  if not client_confs[cluster]:\n    return dict()\n\n  # tell the user which definition that we are using\n  if not cl_args.get('service_url', None):\n    Log.debug(\"Using cluster definition from file %s\" \\\n        % cliconfig.get_cluster_config_file(cluster))\n  else:\n    Log.debug(\"Using cluster service url %s\" % cl_args['service_url'])\n\n  # if cluster definition exists, but service_url is not set, it is an error\n  if not 'service_url' in client_confs[cluster]:\n    config_file = cliconfig.get_cluster_config_file(cluster)\n    Log.error('No service url for %s cluster in %s', cluster, config_file)\n    sys.exit(1)\n\n  # get overrides\n  if 'config_property' in cl_args:\n    pass\n\n  try:\n    cluster_role_env = (cl_args['cluster'], cl_args['role'], cl_args['environ'])\n    config.server_mode_cluster_role_env(cluster_role_env, client_confs)\n    cluster_tuple = config.defaults_cluster_role_env(cluster_role_env)\n  except Exception as ex:\n    Log.error(\"Argument cluster/[role]/[env] is not correct: %s\", str(ex))\n    sys.exit(1)\n\n  new_cl_args = dict()\n  new_cl_args['cluster'] = cluster_tuple[0]\n  new_cl_args['role'] = cluster_tuple[1]\n  new_cl_args['environ'] = cluster_tuple[2]\n  new_cl_args['service_url'] = client_confs[cluster]['service_url'].rstrip('/')\n  new_cl_args['deploy_mode'] = config.SERVER_MODE\n\n  cl_args.update(new_cl_args)\n  return cl_args"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef direct_deployment_mode(command, parser, cluster, cl_args):\n  '''\n  check the direct deployment mode for the given cluster\n  if it is valid return the valid set of args\n  :param command:\n  :param parser:\n  :param cluster:\n  :param cl_args:\n  :return:\n  '''\n\n  cluster = cl_args['cluster']\n  try:\n    config_path = cl_args['config_path']\n    override_config_file = config.parse_override_config_and_write_file(cl_args['config_property'])\n  except KeyError:\n    # if some of the arguments are not found, print error and exit\n    subparser = config.get_subparser(parser, command)\n    print(subparser.format_help())\n    return dict()\n\n  # check if the cluster config directory exists\n  if not cdefs.check_direct_mode_cluster_definition(cluster, config_path):\n    Log.error(\"Cluster config directory \\'%s\\' does not exist\", config_path)\n    return dict()\n\n  config_path = config.get_heron_cluster_conf_dir(cluster, config_path)\n  if not os.path.isdir(config_path):\n    Log.error(\"Cluster config directory \\'%s\\' does not exist\", config_path)\n    return dict()\n\n  Log.info(\"Using cluster definition in %s\" % config_path)\n\n  try:\n    cluster_role_env = (cl_args['cluster'], cl_args['role'], cl_args['environ'])\n    config.direct_mode_cluster_role_env(cluster_role_env, config_path)\n    cluster_tuple = config.defaults_cluster_role_env(cluster_role_env)\n  except Exception as ex:\n    Log.error(\"Argument cluster/[role]/[env] is not correct: %s\", str(ex))\n    return dict()\n\n  new_cl_args = dict()\n  new_cl_args['cluster'] = cluster_tuple[0]\n  new_cl_args['role'] = cluster_tuple[1]\n  new_cl_args['environ'] = cluster_tuple[2]\n  new_cl_args['config_path'] = config_path\n  new_cl_args['override_config_file'] = override_config_file\n  new_cl_args['deploy_mode'] = config.DIRECT_MODE\n\n  cl_args.update(new_cl_args)\n  return cl_args", "response": "check if direct deployment mode for the given cluster"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef extract_common_args(command, parser, cl_args):\n  '''\n  Extract all the common args for all commands\n  :param command:\n  :param parser:\n  :param cl_args:\n  :return:\n  '''\n  try:\n    cluster_role_env = cl_args.pop('cluster/[role]/[env]')\n  except KeyError:\n    try:\n      cluster_role_env = cl_args.pop('cluster')  # for version command\n    except KeyError:\n      # if some of the arguments are not found, print error and exit\n      subparser = config.get_subparser(parser, command)\n      print(subparser.format_help())\n      return dict()\n\n  new_cl_args = dict()\n  cluster_tuple = config.get_cluster_role_env(cluster_role_env)\n  new_cl_args['cluster'] = cluster_tuple[0]\n  new_cl_args['role'] = cluster_tuple[1]\n  new_cl_args['environ'] = cluster_tuple[2]\n  new_cl_args['submit_user'] = getpass.getuser()\n\n  cl_args.update(new_cl_args)\n  return cl_args", "response": "Extract all the common args for all commands\n "}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nexecute the command line and return the exit code.", "response": "def execute(handlers, local_commands):\n  '''\n  Run the command\n  :return:\n  '''\n  # verify if the environment variables are correctly set\n  check_environment()\n\n  # create the argument parser\n  parser = create_parser(handlers)\n\n  # if no argument is provided, print help and exit\n  if len(sys.argv[1:]) == 0:\n    parser.print_help()\n    return 0\n\n  # insert the boolean values for some of the options\n  sys.argv = config.insert_bool_values(sys.argv)\n\n  try:\n    # parse the args\n    args, unknown_args = parser.parse_known_args()\n  except ValueError as ex:\n    Log.error(\"Error while parsing arguments: %s\", str(ex))\n    Log.debug(traceback.format_exc())\n    sys.exit(1)\n\n  command_line_args = vars(args)\n\n  # set log level\n  log.set_logging_level(command_line_args)\n  Log.debug(\"Input Command Line Args: %s\", command_line_args)\n\n# command to be execute\n  command = command_line_args['subcommand']\n  is_local_command = command in local_commands\n\n  if command == 'version':\n    results = run(handlers, command, parser, command_line_args, unknown_args)\n    return 0 if result.is_successful(results) else 1\n\n  if not is_local_command:\n    log.set_logging_level(command_line_args)\n    Log.debug(\"Input Command Line Args: %s\", command_line_args)\n\n    # determine the mode of deployment\n    command_line_args = extract_common_args(command, parser, command_line_args)\n    command_line_args = deployment_mode(command, parser, command_line_args)\n\n    # bail out if args are empty\n    if not command_line_args:\n      return 1\n\n    # register dirs cleanup function during exit\n    if command_line_args['deploy_mode'] == config.DIRECT_MODE and command != \"version\":\n      cleaned_up_files.append(command_line_args['override_config_file'])\n      atexit.register(cleanup, cleaned_up_files)\n\n  # print the input parameters, if verbose is enabled\n  Log.debug(\"Processed Command Line Args: %s\", command_line_args)\n\n  start = time.time()\n  results = run(handlers, command, parser, command_line_args, unknown_args)\n  if not is_local_command:\n    result.render(results)\n  end = time.time()\n\n  if not is_local_command:\n    sys.stdout.flush()\n    Log.debug('Elapsed time: %.3fs.', (end - start))\n\n  return 0 if result.is_successful(results) else 1"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget method returns a sequence of metrics", "response": "def get(self):\n    \"\"\" get method \"\"\"\n    try:\n      cluster = self.get_argument_cluster()\n      role = self.get_argument_role()\n      environ = self.get_argument_environ()\n      topology_name = self.get_argument_topology()\n      component = self.get_argument_component()\n      metric_names = self.get_required_arguments_metricnames()\n\n      topology = self.tracker.getTopologyByClusterRoleEnvironAndName(\n          cluster, role, environ, topology_name)\n\n      interval = int(self.get_argument(constants.PARAM_INTERVAL, default=-1))\n      instances = self.get_arguments(constants.PARAM_INSTANCE)\n\n      metrics = yield tornado.gen.Task(\n          self.getComponentMetrics,\n          topology.tmaster, component, metric_names, instances, interval)\n\n      self.write_success_response(metrics)\n    except Exception as e:\n      Log.debug(traceback.format_exc())\n      self.write_error_response(e)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget the specified metrics for the given component name of this Tmaster.", "response": "def getComponentMetrics(self,\n                          tmaster,\n                          componentName,\n                          metricNames,\n                          instances,\n                          interval,\n                          callback=None):\n    \"\"\"\n    Get the specified metrics for the given component name of this topology.\n    Returns the following dict on success:\n    {\n      \"metrics\": {\n        <metricname>: {\n          <instance>: <numeric value>,\n          <instance>: <numeric value>,\n          ...\n        }, ...\n      },\n      \"interval\": <numeric value>,\n      \"component\": \"...\"\n    }\n\n    Raises exception on failure.\n    \"\"\"\n    if not tmaster or not tmaster.host or not tmaster.stats_port:\n      raise Exception(\"No Tmaster found\")\n\n    host = tmaster.host\n    port = tmaster.stats_port\n\n    metricRequest = tmaster_pb2.MetricRequest()\n    metricRequest.component_name = componentName\n    if len(instances) > 0:\n      for instance in instances:\n        metricRequest.instance_id.append(instance)\n    for metricName in metricNames:\n      metricRequest.metric.append(metricName)\n    metricRequest.interval = interval\n\n    # Serialize the metricRequest to send as a payload\n    # with the HTTP request.\n    metricRequestString = metricRequest.SerializeToString()\n\n    url = \"http://{0}:{1}/stats\".format(host, port)\n    request = tornado.httpclient.HTTPRequest(url,\n                                             body=metricRequestString,\n                                             method='POST',\n                                             request_timeout=5)\n\n    Log.debug(\"Making HTTP call to fetch metrics\")\n    Log.debug(\"url: \" + url)\n    try:\n      client = tornado.httpclient.AsyncHTTPClient()\n      result = yield client.fetch(request)\n      Log.debug(\"HTTP call complete.\")\n    except tornado.httpclient.HTTPError as e:\n      raise Exception(str(e))\n\n    # Check the response code - error if it is in 400s or 500s\n    responseCode = result.code\n    if responseCode >= 400:\n      message = \"Error in getting metrics from Tmaster, code: \" + responseCode\n      Log.error(message)\n      raise Exception(message)\n\n    # Parse the response from tmaster.\n    metricResponse = tmaster_pb2.MetricResponse()\n    metricResponse.ParseFromString(result.body)\n\n    if metricResponse.status.status == common_pb2.NOTOK:\n      if metricResponse.status.HasField(\"message\"):\n        Log.warn(\"Received response from Tmaster: %s\", metricResponse.status.message)\n\n    # Form the response.\n    ret = {}\n    ret[\"interval\"] = metricResponse.interval\n    ret[\"component\"] = componentName\n    ret[\"metrics\"] = {}\n    for metric in metricResponse.metric:\n      instance = metric.instance_id\n      for im in metric.metric:\n        metricname = im.name\n        value = im.value\n        if metricname not in ret[\"metrics\"]:\n          ret[\"metrics\"][metricname] = {}\n        ret[\"metrics\"][metricname][instance] = value\n\n    raise tornado.gen.Return(ret)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef initialize(self, config, context):\n    if SlidingWindowBolt.WINDOW_DURATION_SECS in config:\n      self.window_duration = int(config[SlidingWindowBolt.WINDOW_DURATION_SECS])\n    else:\n      self.logger.fatal(\"Window Duration has to be specified in the config\")\n    if SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS in config:\n      self.slide_interval = int(config[SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS])\n    else:\n      self.slide_interval = self.window_duration\n    if self.slide_interval > self.window_duration:\n      self.logger.fatal(\"Slide Interval should be <= Window Duration\")\n\n    # By modifying the config, we are able to setup the tick timer\n    config[api_constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS] = str(self.slide_interval)\n    self.current_tuples = deque()\n    if hasattr(self, 'saved_state'):\n      if 'tuples' in self.saved_state:\n        self.current_tuples = self.saved_state['tuples']", "response": "Initialize the current window duration and slide interval"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nprocessing a single tuple of input We add the tuple pair into our current_tuples and then expire the elemnents .", "response": "def process(self, tup):\n    \"\"\"Process a single tuple of input\n\n    We add the (time, tuple) pair into our current_tuples. And then look for expiring\n    elemnents\n    \"\"\"\n    curtime = int(time.time())\n    self.current_tuples.append((tup, curtime))\n    self._expire(curtime)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing the window duration and slide interval", "response": "def initialize(self, config, context):\n    \"\"\"We initialize the window duration and slide interval\n    \"\"\"\n    if TumblingWindowBolt.WINDOW_DURATION_SECS in config:\n      self.window_duration = int(config[TumblingWindowBolt.WINDOW_DURATION_SECS])\n    else:\n      self.logger.fatal(\"Window Duration has to be specified in the config\")\n\n    # By modifying the config, we are able to setup the tick timer\n    config[api_constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS] = str(self.window_duration)\n    self.current_tuples = deque()\n    if hasattr(self, 'saved_state'):\n      if 'tuples' in self.saved_state:\n        self.current_tuples = self.saved_state['tuples']"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get(self, path):\n\n    logging.debug(\"request to download: %s\", path)\n    # If the file is large, we want to abandon downloading\n    # if user cancels the requests.\n    # pylint: disable=attribute-defined-outside-init\n    self.connection_closed = False\n\n    self.set_header(\"Content-Disposition\", \"attachment\")\n    if not utils.check_path(path):\n      self.write(\"Only relative paths are allowed\")\n      self.set_status(403)\n      self.finish()\n      return\n\n    if path is None or not os.path.isfile(path):\n      self.write(\"File %s  not found\" % path)\n      self.set_status(404)\n      self.finish()\n      return\n\n    length = int(4 * 1024 * 1024)\n    offset = int(0)\n    while True:\n      data = utils.read_chunk(path, offset=offset, length=length, escape_data=False)\n      if self.connection_closed or 'data' not in data or len(data['data']) < length:\n        break\n      offset += length\n      self.write(data['data'])\n      self.flush()\n\n    if 'data' in data:\n      self.write(data['data'])\n    self.finish()", "response": "get method for the base class"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget summary of stream managers registration summary", "response": "def getStmgrsRegSummary(self, tmaster, callback=None):\n    \"\"\"\n    Get summary of stream managers registration summary\n    \"\"\"\n    if not tmaster or not tmaster.host or not tmaster.stats_port:\n      return\n    reg_request = tmaster_pb2.StmgrsRegistrationSummaryRequest()\n    request_str = reg_request.SerializeToString()\n    port = str(tmaster.stats_port)\n    host = tmaster.host\n    url = \"http://{0}:{1}/stmgrsregistrationsummary\".format(host, port)\n    request = tornado.httpclient.HTTPRequest(url,\n                                             body=request_str,\n                                             method='POST',\n                                             request_timeout=5)\n    Log.debug('Making HTTP call to fetch stmgrsregistrationsummary url: %s', url)\n    try:\n      client = tornado.httpclient.AsyncHTTPClient()\n      result = yield client.fetch(request)\n      Log.debug(\"HTTP call complete.\")\n    except tornado.httpclient.HTTPError as e:\n      raise Exception(str(e))\n    # Check the response code - error if it is in 400s or 500s\n    responseCode = result.code\n    if responseCode >= 400:\n      message = \"Error in getting exceptions from Tmaster, code: \" + responseCode\n      Log.error(message)\n      raise tornado.gen.Return({\n          \"message\": message\n      })\n    # Parse the response from tmaster.\n    reg_response = tmaster_pb2.StmgrsRegistrationSummaryResponse()\n    reg_response.ParseFromString(result.body)\n    # Send response\n    ret = {}\n    for stmgr in reg_response.registered_stmgrs:\n      ret[stmgr] = True\n    for stmgr in reg_response.absent_stmgrs:\n      ret[stmgr] = False\n    raise tornado.gen.Return(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get(self):\n    try:\n      cluster = self.get_argument_cluster()\n      role = self.get_argument_role()\n      environ = self.get_argument_environ()\n      topology_name = self.get_argument_topology()\n      topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)\n      runtime_state = topology_info[\"runtime_state\"]\n      runtime_state[\"topology_version\"] = topology_info[\"metadata\"][\"release_version\"]\n      topology = self.tracker.getTopologyByClusterRoleEnvironAndName(\n          cluster, role, environ, topology_name)\n      reg_summary = yield tornado.gen.Task(self.getStmgrsRegSummary, topology.tmaster)\n      for stmgr, reg in reg_summary.items():\n        runtime_state[\"stmgrs\"].setdefault(stmgr, {})[\"is_registered\"] = reg\n      self.write_success_response(runtime_state)\n    except Exception as e:\n      Log.debug(traceback.format_exc())\n      self.write_error_response(e)", "response": "get method returns a generator that yields all the services that are registered"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nwrite content to a file and then rename it to target path", "response": "def atomic_write_file(path, content):\n  \"\"\"\n  file.write(...) is not atomic.\n  We write to a tmp file and then rename to target path since rename is atomic.\n  We do this to avoid the content of file is dirty read/partially read by others.\n  \"\"\"\n  # Write to a randomly tmp file\n  tmp_file = get_tmp_filename()\n  with open(tmp_file, 'w') as f:\n    f.write(content)\n    # make sure that all data is on disk\n    f.flush()\n    os.fsync(f.fileno())\n\n  # Rename the tmp file\n  os.rename(tmp_file, path)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting up log, process and signal handlers", "response": "def setup(executor):\n  \"\"\"Set up log, process and signal handlers\"\"\"\n  # pylint: disable=unused-argument\n  def signal_handler(signal_to_handle, frame):\n    # We would do nothing here but just exit\n    # Just catch the SIGTERM and then cleanup(), registered with atexit, would invoke\n    Log.info('signal_handler invoked with signal %s', signal_to_handle)\n    executor.stop_state_manager_watches()\n    sys.exit(signal_to_handle)\n\n  def cleanup():\n    \"\"\"Handler to trigger when receiving the SIGTERM signal\n    Do cleanup inside this method, including:\n    1. Terminate all children processes\n    \"\"\"\n    Log.info('Executor terminated; exiting all process in executor.')\n\n    # Kill child processes first and wait for log collection to finish\n    for pid in executor.processes_to_monitor.keys():\n      os.kill(pid, signal.SIGTERM)\n    time.sleep(5)\n\n    # We would not wait or check whether process spawned dead or not\n    os.killpg(0, signal.SIGTERM)\n\n  # Redirect stdout and stderr to files in append mode\n  # The filename format is heron-executor-<container_id>.stdxxx\n  shardid = executor.shard\n  log.configure(logfile='heron-executor-%s.stdout' % shardid)\n\n  pid = os.getpid()\n  sid = os.getsid(pid)\n\n  # POSIX prohibits the change of the process group ID of a session leader\n  if pid <> sid:\n    Log.info('Set up process group; executor becomes leader')\n    os.setpgrp() # create new process group, become its leader\n\n  Log.info('Register the SIGTERM signal handler')\n  signal.signal(signal.SIGTERM, signal_handler)\n\n  Log.info('Register the atexit clean up')\n  atexit.register(cleanup)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef main():\n  # Since Heron on YARN runs as headless users, pex compiled\n  # binaries should be exploded into the container working\n  # directory. In order to do this, we need to set the\n  # PEX_ROOT shell environment before forking the processes\n  shell_env = os.environ.copy()\n  shell_env[\"PEX_ROOT\"] = os.path.join(os.path.abspath('.'), \".pex\")\n\n  # Instantiate the executor, bind it to signal handlers and launch it\n  executor = HeronExecutor(sys.argv, shell_env)\n  executor.initialize()\n\n  start(executor)", "response": "Register exit handlers initialize the executor and run it."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninitializing from parsed arguments", "response": "def init_from_parsed_args(self, parsed_args):\n    \"\"\" initialize from parsed arguments \"\"\"\n    self.shard = parsed_args.shard\n    self.topology_name = parsed_args.topology_name\n    self.topology_id = parsed_args.topology_id\n    self.topology_defn_file = parsed_args.topology_defn_file\n    self.state_manager_connection = parsed_args.state_manager_connection\n    self.state_manager_root = parsed_args.state_manager_root\n    self.state_manager_config_file = parsed_args.state_manager_config_file\n    self.tmaster_binary = parsed_args.tmaster_binary\n    self.stmgr_binary = parsed_args.stmgr_binary\n    self.metrics_manager_classpath = parsed_args.metrics_manager_classpath\n    self.metricscache_manager_classpath = parsed_args.metricscache_manager_classpath\n    # '=' can be parsed in a wrong way by some schedulers (aurora) hence it needs to be escaped.\n    # It is escaped in two different ways. '(61)' is the new escaping. '&equals;' was\n    # the original replacement but it is not friendly to bash and is causing issues. The original\n    # escaping is still left there for reference and backward compatibility purposes (to be\n    # removed after no topology needs it)\n    self.instance_jvm_opts =\\\n        base64.b64decode(parsed_args.instance_jvm_opts.lstrip('\"').\n                         rstrip('\"').replace('(61)', '=').replace('&equals;', '='))\n    self.classpath = parsed_args.classpath\n    # Needed for Docker environments since the hostname of a docker container is the container's\n    # id within docker, rather than the host's hostname. NOTE: this 'HOST' env variable is not\n    # guaranteed to be set in all Docker executor environments (outside of Marathon)\n    if is_docker_environment():\n      self.master_host = os.environ.get('HOST') if 'HOST' in os.environ else socket.gethostname()\n    else:\n      self.master_host = socket.gethostname()\n    self.master_port = parsed_args.master_port\n    self.tmaster_controller_port = parsed_args.tmaster_controller_port\n    self.tmaster_stats_port = parsed_args.tmaster_stats_port\n    self.heron_internals_config_file = parsed_args.heron_internals_config_file\n    self.override_config_file = parsed_args.override_config_file\n    self.component_ram_map =\\\n        map(lambda x: {x.split(':')[0]:\n                           int(x.split(':')[1])}, parsed_args.component_ram_map.split(','))\n    self.component_ram_map =\\\n        functools.reduce(lambda x, y: dict(x.items() + y.items()), self.component_ram_map)\n\n    # component_jvm_opts_in_base64 itself is a base64-encoding-json-map, which is appended with\n    # \" at the start and end. It also escapes \"=\" to \"&equals\" due to aurora limitation\n    # And the json is a map from base64-encoding-component-name to base64-encoding-jvm-options\n    self.component_jvm_opts = {}\n    # First we need to decode the base64 string back to a json map string.\n    # '=' can be parsed in a wrong way by some schedulers (aurora) hence it needs to be escaped.\n    # It is escaped in two different ways. '(61)' is the new escaping. '&equals;' was\n    # the original replacement but it is not friendly to bash and is causing issues. The original\n    # escaping is still left there for reference and backward compatibility purposes (to be\n    # removed after no topology needs it)\n    component_jvm_opts_in_json =\\\n        base64.b64decode(parsed_args.component_jvm_opts.\n                         lstrip('\"').rstrip('\"').replace('(61)', '=').replace('&equals;', '='))\n    if component_jvm_opts_in_json != \"\":\n      for (k, v) in json.loads(component_jvm_opts_in_json).items():\n        # In json, the component name and JVM options are still in base64 encoding\n        self.component_jvm_opts[base64.b64decode(k)] = base64.b64decode(v)\n\n    self.pkg_type = parsed_args.pkg_type\n    self.topology_binary_file = parsed_args.topology_binary_file\n    self.heron_java_home = parsed_args.heron_java_home\n    self.shell_port = parsed_args.shell_port\n    self.heron_shell_binary = parsed_args.heron_shell_binary\n    self.metrics_manager_port = parsed_args.metrics_manager_port\n    self.metricscache_manager_master_port = parsed_args.metricscache_manager_master_port\n    self.metricscache_manager_stats_port = parsed_args.metricscache_manager_stats_port\n    self.cluster = parsed_args.cluster\n    self.role = parsed_args.role\n    self.environment = parsed_args.environment\n    self.instance_classpath = parsed_args.instance_classpath\n    self.metrics_sinks_config_file = parsed_args.metrics_sinks_config_file\n    self.scheduler_classpath = parsed_args.scheduler_classpath\n    self.scheduler_port = parsed_args.scheduler_port\n    self.python_instance_binary = parsed_args.python_instance_binary\n    self.cpp_instance_binary = parsed_args.cpp_instance_binary\n\n    self.is_stateful_topology = (parsed_args.is_stateful.lower() == 'true')\n    self.checkpoint_manager_classpath = parsed_args.checkpoint_manager_classpath\n    self.checkpoint_manager_port = parsed_args.checkpoint_manager_port\n    self.checkpoint_manager_ram = parsed_args.checkpoint_manager_ram\n    self.stateful_config_file = parsed_args.stateful_config_file\n    self.metricscache_manager_mode = parsed_args.metricscache_manager_mode \\\n        if parsed_args.metricscache_manager_mode else \"disabled\"\n    self.health_manager_mode = parsed_args.health_manager_mode\n    self.health_manager_classpath = '%s:%s'\\\n        % (self.scheduler_classpath, parsed_args.health_manager_classpath)\n    self.jvm_remote_debugger_ports = \\\n      parsed_args.jvm_remote_debugger_ports.split(\",\") \\\n        if parsed_args.jvm_remote_debugger_ports else None"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parse_args(args):\n    Log.info(\"Input args: %r\" % args)\n\n    parser = argparse.ArgumentParser()\n\n    parser.add_argument(\"--shard\", type=int, required=True)\n    parser.add_argument(\"--topology-name\", required=True)\n    parser.add_argument(\"--topology-id\", required=True)\n    parser.add_argument(\"--topology-defn-file\", required=True)\n    parser.add_argument(\"--state-manager-connection\", required=True)\n    parser.add_argument(\"--state-manager-root\", required=True)\n    parser.add_argument(\"--state-manager-config-file\", required=True)\n    parser.add_argument(\"--tmaster-binary\", required=True)\n    parser.add_argument(\"--stmgr-binary\", required=True)\n    parser.add_argument(\"--metrics-manager-classpath\", required=True)\n    parser.add_argument(\"--instance-jvm-opts\", required=True)\n    parser.add_argument(\"--classpath\", required=True)\n    parser.add_argument(\"--master-port\", required=True)\n    parser.add_argument(\"--tmaster-controller-port\", required=True)\n    parser.add_argument(\"--tmaster-stats-port\", required=True)\n    parser.add_argument(\"--heron-internals-config-file\", required=True)\n    parser.add_argument(\"--override-config-file\", required=True)\n    parser.add_argument(\"--component-ram-map\", required=True)\n    parser.add_argument(\"--component-jvm-opts\", required=True)\n    parser.add_argument(\"--pkg-type\", required=True)\n    parser.add_argument(\"--topology-binary-file\", required=True)\n    parser.add_argument(\"--heron-java-home\", required=True)\n    parser.add_argument(\"--shell-port\", required=True)\n    parser.add_argument(\"--heron-shell-binary\", required=True)\n    parser.add_argument(\"--metrics-manager-port\", required=True)\n    parser.add_argument(\"--cluster\", required=True)\n    parser.add_argument(\"--role\", required=True)\n    parser.add_argument(\"--environment\", required=True)\n    parser.add_argument(\"--instance-classpath\", required=True)\n    parser.add_argument(\"--metrics-sinks-config-file\", required=True)\n    parser.add_argument(\"--scheduler-classpath\", required=True)\n    parser.add_argument(\"--scheduler-port\", required=True)\n    parser.add_argument(\"--python-instance-binary\", required=True)\n    parser.add_argument(\"--cpp-instance-binary\", required=True)\n    parser.add_argument(\"--metricscache-manager-classpath\", required=True)\n    parser.add_argument(\"--metricscache-manager-master-port\", required=True)\n    parser.add_argument(\"--metricscache-manager-stats-port\", required=True)\n    parser.add_argument(\"--metricscache-manager-mode\", required=False)\n    parser.add_argument(\"--is-stateful\", required=True)\n    parser.add_argument(\"--checkpoint-manager-classpath\", required=True)\n    parser.add_argument(\"--checkpoint-manager-port\", required=True)\n    parser.add_argument(\"--checkpoint-manager-ram\", type=long, required=True)\n    parser.add_argument(\"--stateful-config-file\", required=True)\n    parser.add_argument(\"--health-manager-mode\", required=True)\n    parser.add_argument(\"--health-manager-classpath\", required=True)\n    parser.add_argument(\"--jvm-remote-debugger-ports\", required=False,\n                        help=\"ports to be used by a remote debugger for JVM instances\")\n\n    parsed_args, unknown_args = parser.parse_known_args(args[1:])\n\n    if unknown_args:\n      Log.error('Unknown argument: %s' % unknown_args[0])\n      parser.print_help()\n      sys.exit(1)\n\n    return parsed_args", "response": "Uses argparse to collect positional args"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef initialize(self):\n    create_folders = Command('mkdir -p %s' % self.log_dir, self.shell_env)\n    self.run_command_or_exit(create_folders)\n\n    chmod_logs_dir = Command('chmod a+rx . && chmod a+x %s' % self.log_dir, self.shell_env)\n    self.run_command_or_exit(chmod_logs_dir)\n\n    chmod_x_binaries = [self.tmaster_binary, self.stmgr_binary, self.heron_shell_binary]\n\n    for binary in chmod_x_binaries:\n      stat_result = os.stat(binary)[stat.ST_MODE]\n      if not stat_result & stat.S_IXOTH:\n        chmod_binary = Command('chmod +x %s' % binary, self.shell_env)\n        self.run_command_or_exit(chmod_binary)\n\n    # Log itself pid\n    log_pid_for_process(get_heron_executor_process_name(self.shard), os.getpid())", "response": "Initialize the environment. Done with a method call outside of the constructor for 2 reasons:\n    1. Unit tests probably won't want/need to do this\n    2. We don't initialize the logger (also something unit tests don't want) until after the\n    constructor"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the command to start the metrics manager processes", "response": "def _get_metricsmgr_cmd(self, metricsManagerId, sink_config_file, port):\n    ''' get the command to start the metrics manager processes '''\n    metricsmgr_main_class = 'org.apache.heron.metricsmgr.MetricsManager'\n\n    metricsmgr_cmd = [os.path.join(self.heron_java_home, 'bin/java'),\n                      # We could not rely on the default -Xmx setting, which could be very big,\n                      # for instance, the default -Xmx in Twitter mesos machine is around 18GB\n                      '-Xmx1024M',\n                      '-XX:+PrintCommandLineFlags',\n                      '-verbosegc',\n                      '-XX:+PrintGCDetails',\n                      '-XX:+PrintGCTimeStamps',\n                      '-XX:+PrintGCDateStamps',\n                      '-XX:+PrintGCCause',\n                      '-XX:+UseGCLogFileRotation',\n                      '-XX:NumberOfGCLogFiles=5',\n                      '-XX:GCLogFileSize=100M',\n                      '-XX:+PrintPromotionFailure',\n                      '-XX:+PrintTenuringDistribution',\n                      '-XX:+PrintHeapAtGC',\n                      '-XX:+HeapDumpOnOutOfMemoryError',\n                      '-XX:+UseConcMarkSweepGC',\n                      '-XX:+PrintCommandLineFlags',\n                      '-Xloggc:log-files/gc.metricsmgr.log',\n                      '-Djava.net.preferIPv4Stack=true',\n                      '-cp',\n                      self.metrics_manager_classpath,\n                      metricsmgr_main_class,\n                      '--id=' + metricsManagerId,\n                      '--port=' + str(port),\n                      '--topology=' + self.topology_name,\n                      '--cluster=' + self.cluster,\n                      '--role=' + self.role,\n                      '--environment=' + self.environment,\n                      '--topology-id=' + self.topology_id,\n                      '--system-config-file=' + self.heron_internals_config_file,\n                      '--override-config-file=' + self.override_config_file,\n                      '--sink-config-file=' + sink_config_file]\n\n    return Command(metricsmgr_cmd, self.shell_env)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the command to start the metrics manager processes", "response": "def _get_metrics_cache_cmd(self):\n    ''' get the command to start the metrics manager processes '''\n    metricscachemgr_main_class = 'org.apache.heron.metricscachemgr.MetricsCacheManager'\n\n    metricscachemgr_cmd = [os.path.join(self.heron_java_home, 'bin/java'),\n                           # We could not rely on the default -Xmx setting, which could be very big,\n                           # for instance, the default -Xmx in Twitter mesos machine is around 18GB\n                           '-Xmx1024M',\n                           '-XX:+PrintCommandLineFlags',\n                           '-verbosegc',\n                           '-XX:+PrintGCDetails',\n                           '-XX:+PrintGCTimeStamps',\n                           '-XX:+PrintGCDateStamps',\n                           '-XX:+PrintGCCause',\n                           '-XX:+UseGCLogFileRotation',\n                           '-XX:NumberOfGCLogFiles=5',\n                           '-XX:GCLogFileSize=100M',\n                           '-XX:+PrintPromotionFailure',\n                           '-XX:+PrintTenuringDistribution',\n                           '-XX:+PrintHeapAtGC',\n                           '-XX:+HeapDumpOnOutOfMemoryError',\n                           '-XX:+UseConcMarkSweepGC',\n                           '-XX:+PrintCommandLineFlags',\n                           '-Xloggc:log-files/gc.metricscache.log',\n                           '-Djava.net.preferIPv4Stack=true',\n                           '-cp',\n                           self.metricscache_manager_classpath,\n                           metricscachemgr_main_class,\n                           \"--metricscache_id\", 'metricscache-0',\n                           \"--master_port\", self.metricscache_manager_master_port,\n                           \"--stats_port\", self.metricscache_manager_stats_port,\n                           \"--topology_name\", self.topology_name,\n                           \"--topology_id\", self.topology_id,\n                           \"--system_config_file\", self.heron_internals_config_file,\n                           \"--override_config_file\", self.override_config_file,\n                           \"--sink_config_file\", self.metrics_sinks_config_file,\n                           \"--cluster\", self.cluster,\n                           \"--role\", self.role,\n                           \"--environment\", self.environment]\n\n    return Command(metricscachemgr_cmd, self.shell_env)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _get_healthmgr_cmd(self):\n    ''' get the command to start the topology health manager processes '''\n    healthmgr_main_class = 'org.apache.heron.healthmgr.HealthManager'\n\n    healthmgr_cmd = [os.path.join(self.heron_java_home, 'bin/java'),\n                     # We could not rely on the default -Xmx setting, which could be very big,\n                     # for instance, the default -Xmx in Twitter mesos machine is around 18GB\n                     '-Xmx1024M',\n                     '-XX:+PrintCommandLineFlags',\n                     '-verbosegc',\n                     '-XX:+PrintGCDetails',\n                     '-XX:+PrintGCTimeStamps',\n                     '-XX:+PrintGCDateStamps',\n                     '-XX:+PrintGCCause',\n                     '-XX:+UseGCLogFileRotation',\n                     '-XX:NumberOfGCLogFiles=5',\n                     '-XX:GCLogFileSize=100M',\n                     '-XX:+PrintPromotionFailure',\n                     '-XX:+PrintTenuringDistribution',\n                     '-XX:+PrintHeapAtGC',\n                     '-XX:+HeapDumpOnOutOfMemoryError',\n                     '-XX:+UseConcMarkSweepGC',\n                     '-XX:+PrintCommandLineFlags',\n                     '-Xloggc:log-files/gc.healthmgr.log',\n                     '-Djava.net.preferIPv4Stack=true',\n                     '-cp', self.health_manager_classpath,\n                     healthmgr_main_class,\n                     \"--cluster\", self.cluster,\n                     \"--role\", self.role,\n                     \"--environment\", self.environment,\n                     \"--topology_name\", self.topology_name,\n                     \"--metricsmgr_port\", self.metrics_manager_port]\n\n    return Command(healthmgr_cmd, self.shell_env)", "response": "get the command to start the topology health manager processes"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _get_tmaster_processes(self):\n    ''' get the command to start the tmaster processes '''\n    retval = {}\n    tmaster_cmd_lst = [\n        self.tmaster_binary,\n        '--topology_name=%s' % self.topology_name,\n        '--topology_id=%s' % self.topology_id,\n        '--zkhostportlist=%s' % self.state_manager_connection,\n        '--zkroot=%s' % self.state_manager_root,\n        '--myhost=%s' % self.master_host,\n        '--master_port=%s' % str(self.master_port),\n        '--controller_port=%s' % str(self.tmaster_controller_port),\n        '--stats_port=%s' % str(self.tmaster_stats_port),\n        '--config_file=%s' % self.heron_internals_config_file,\n        '--override_config_file=%s' % self.override_config_file,\n        '--metrics_sinks_yaml=%s' % self.metrics_sinks_config_file,\n        '--metricsmgr_port=%s' % str(self.metrics_manager_port),\n        '--ckptmgr_port=%s' % str(self.checkpoint_manager_port)]\n\n    tmaster_env = self.shell_env.copy() if self.shell_env is not None else {}\n    tmaster_cmd = Command(tmaster_cmd_lst, tmaster_env)\n    if os.environ.get('ENABLE_HEAPCHECK') is not None:\n      tmaster_cmd.env.update({\n          'LD_PRELOAD': \"/usr/lib/libtcmalloc.so\",\n          'HEAPCHECK': \"normal\"\n      })\n\n    retval[\"heron-tmaster\"] = tmaster_cmd\n\n    if self.metricscache_manager_mode.lower() != \"disabled\":\n      retval[\"heron-metricscache\"] = self._get_metrics_cache_cmd()\n\n    if self.health_manager_mode.lower() != \"disabled\":\n      retval[\"heron-healthmgr\"] = self._get_healthmgr_cmd()\n\n    retval[self.metricsmgr_ids[0]] = self._get_metricsmgr_cmd(\n        self.metricsmgr_ids[0],\n        self.metrics_sinks_config_file,\n        self.metrics_manager_port)\n\n    if self.is_stateful_topology:\n      retval.update(self._get_ckptmgr_process())\n\n    return retval", "response": "get the command to start the tmaster processes"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _get_ckptmgr_process(self):\n    ''' Get the command to start the checkpoint manager process'''\n\n    ckptmgr_main_class = 'org.apache.heron.ckptmgr.CheckpointManager'\n\n    ckptmgr_ram_mb = self.checkpoint_manager_ram / (1024 * 1024)\n    ckptmgr_cmd = [os.path.join(self.heron_java_home, \"bin/java\"),\n                   '-Xms%dM' % ckptmgr_ram_mb,\n                   '-Xmx%dM' % ckptmgr_ram_mb,\n                   '-XX:+PrintCommandLineFlags',\n                   '-verbosegc',\n                   '-XX:+PrintGCDetails',\n                   '-XX:+PrintGCTimeStamps',\n                   '-XX:+PrintGCDateStamps',\n                   '-XX:+PrintGCCause',\n                   '-XX:+UseGCLogFileRotation',\n                   '-XX:NumberOfGCLogFiles=5',\n                   '-XX:GCLogFileSize=100M',\n                   '-XX:+PrintPromotionFailure',\n                   '-XX:+PrintTenuringDistribution',\n                   '-XX:+PrintHeapAtGC',\n                   '-XX:+HeapDumpOnOutOfMemoryError',\n                   '-XX:+UseConcMarkSweepGC',\n                   '-XX:+UseConcMarkSweepGC',\n                   '-Xloggc:log-files/gc.ckptmgr.log',\n                   '-Djava.net.preferIPv4Stack=true',\n                   '-cp',\n                   self.checkpoint_manager_classpath,\n                   ckptmgr_main_class,\n                   '-t' + self.topology_name,\n                   '-i' + self.topology_id,\n                   '-c' + self.ckptmgr_ids[self.shard],\n                   '-p' + self.checkpoint_manager_port,\n                   '-f' + self.stateful_config_file,\n                   '-o' + self.override_config_file,\n                   '-g' + self.heron_internals_config_file]\n    retval = {}\n    retval[self.ckptmgr_ids[self.shard]] = Command(ckptmgr_cmd, self.shell_env)\n\n    return retval", "response": "Get the command to start the checkpoint manager process"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _get_instance_plans(self, packing_plan, container_id):\n    this_container_plan = None\n    for container_plan in packing_plan.container_plans:\n      if container_plan.id == container_id:\n        this_container_plan = container_plan\n\n    # When the executor runs in newly added container by `heron update`,\n    # there is no plan for this container. In this situation,\n    # return None to bypass instance processes.\n    if this_container_plan is None:\n      return None\n    return this_container_plan.instance_plans", "response": "Get the instance_plans for the given packing_plan."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _get_heron_support_processes(self):\n    retval = {}\n\n    retval[self.heron_shell_ids[self.shard]] = Command([\n        '%s' % self.heron_shell_binary,\n        '--port=%s' % self.shell_port,\n        '--log_file_prefix=%s/heron-shell-%s.log' % (self.log_dir, self.shard),\n        '--secret=%s' % self.topology_id], self.shell_env)\n\n    return retval", "response": "Get a map from all daemon services name to the command to start them"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nwait for the termination of a process and log its stdout & stderr.", "response": "def _wait_process_std_out_err(self, name, process):\n    ''' Wait for the termination of a process and log its stdout & stderr '''\n    proc.stream_process_stdout(process, stdout_log_fn(name))\n    process.wait()"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _start_processes(self, commands):\n    Log.info(\"Start processes\")\n    processes_to_monitor = {}\n    # First start all the processes\n    for (name, command) in commands.items():\n      p = self._run_process(name, command)\n      processes_to_monitor[p.pid] = ProcessInfo(p, name, command)\n\n      # Log down the pid file\n      log_pid_for_process(name, p.pid)\n\n    with self.process_lock:\n      self.processes_to_monitor.update(processes_to_monitor)", "response": "Start all the processes and add them to the processes_to_monitor dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef start_process_monitor(self):\n    # Now wait for any child to die\n    Log.info(\"Start process monitor\")\n    while True:\n      if len(self.processes_to_monitor) > 0:\n        (pid, status) = os.wait()\n\n        with self.process_lock:\n          if pid in self.processes_to_monitor.keys():\n            old_process_info = self.processes_to_monitor[pid]\n            name = old_process_info.name\n            command = old_process_info.command\n            Log.info(\"%s (pid=%s) exited with status %d. command=%s\" % (name, pid, status, command))\n            # Log the stdout & stderr of the failed process\n            self._wait_process_std_out_err(name, old_process_info.process)\n\n            # Just make it world readable\n            if os.path.isfile(\"core.%d\" % pid):\n              os.system(\"chmod a+r core.%d\" % pid)\n            if old_process_info.attempts >= self.max_runs:\n              Log.info(\"%s exited too many times\" % name)\n              sys.exit(1)\n            time.sleep(self.interval_between_runs)\n            p = self._run_process(name, command)\n            del self.processes_to_monitor[pid]\n            self.processes_to_monitor[p.pid] =\\\n              ProcessInfo(p, name, command, old_process_info.attempts + 1)\n\n            # Log down the pid file\n            log_pid_for_process(name, p.pid)", "response": "Start the process monitor."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_commands_to_run(self):\n    # During shutdown the watch might get triggered with the empty packing plan\n    if len(self.packing_plan.container_plans) == 0:\n      return {}\n    if self._get_instance_plans(self.packing_plan, self.shard) is None and self.shard != 0:\n      retval = {}\n      retval['heron-shell'] = Command([\n          '%s' % self.heron_shell_binary,\n          '--port=%s' % self.shell_port,\n          '--log_file_prefix=%s/heron-shell-%s.log' % (self.log_dir, self.shard),\n          '--secret=%s' % self.topology_id], self.shell_env)\n      return retval\n\n    if self.shard == 0:\n      commands = self._get_tmaster_processes()\n    else:\n      self._untar_if_needed()\n      commands = self._get_streaming_processes()\n\n    # Attach daemon processes\n    commands.update(self._get_heron_support_processes())\n    return commands", "response": "Prepare either TMaster or Streaming commands according to shard."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a 3 - tuple of dicts that contains the current command and the updated command that has changed.", "response": "def get_command_changes(self, current_commands, updated_commands):\n    \"\"\"\n    Compares the current command with updated command to return a 3-tuple of dicts,\n    keyed by command name: commands_to_kill, commands_to_keep and commands_to_start.\n    \"\"\"\n    commands_to_kill = {}\n    commands_to_keep = {}\n    commands_to_start = {}\n\n    # if the current command has a matching command in the updated commands we keep it\n    # otherwise we kill it\n    for current_name, current_command in current_commands.items():\n      # We don't restart tmaster since it watches the packing plan and updates itself. The stream\n      # manager is restarted just to reset state, but we could update it to do so without a restart\n      if current_name in updated_commands.keys() and \\\n        current_command == updated_commands[current_name] and \\\n        not current_name.startswith('stmgr-'):\n        commands_to_keep[current_name] = current_command\n      else:\n        commands_to_kill[current_name] = current_command\n\n    # updated commands not in the keep list need to be started\n    for updated_name, updated_command in updated_commands.items():\n      if updated_name not in commands_to_keep.keys():\n        commands_to_start[updated_name] = updated_command\n\n    return commands_to_kill, commands_to_keep, commands_to_start"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nlaunching the processes required by the processes in the process_lock.", "response": "def launch(self):\n    ''' Determines the commands to be run and compares them with the existing running commands.\n    Then starts new ones required and kills old ones no longer required.\n    '''\n    with self.process_lock:\n      current_commands = dict(map((lambda process: (process.name, process.command)),\n                                  self.processes_to_monitor.values()))\n      updated_commands = self.get_commands_to_run()\n\n      # get the commands to kill, keep and start\n      commands_to_kill, commands_to_keep, commands_to_start = \\\n          self.get_command_changes(current_commands, updated_commands)\n\n      Log.info(\"current commands: %s\" % sorted(current_commands.keys()))\n      Log.info(\"new commands    : %s\" % sorted(updated_commands.keys()))\n      Log.info(\"commands_to_kill: %s\" % sorted(commands_to_kill.keys()))\n      Log.info(\"commands_to_keep: %s\" % sorted(commands_to_keep.keys()))\n      Log.info(\"commands_to_start: %s\" % sorted(commands_to_start.keys()))\n\n      self._kill_processes(commands_to_kill)\n      self._start_processes(commands_to_start)\n      Log.info(\"Launch complete - processes killed=%s kept=%s started=%s monitored=%s\" %\n               (len(commands_to_kill), len(commands_to_keep),\n                len(commands_to_start), len(self.processes_to_monitor)))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef start_state_manager_watches(self):\n    Log.info(\"Start state manager watches\")\n    statemgr_config = StateMgrConfig()\n    statemgr_config.set_state_locations(configloader.load_state_manager_locations(\n        self.cluster, state_manager_config_file=self.state_manager_config_file,\n        overrides={\"heron.statemgr.connection.string\": self.state_manager_connection}))\n    try:\n      self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config)\n      for state_manager in self.state_managers:\n        state_manager.start()\n    except Exception as ex:\n      Log.error(\"Found exception while initializing state managers: %s. Bailing out...\" % ex)\n      traceback.print_exc()\n      sys.exit(1)\n\n    # pylint: disable=unused-argument\n    def on_packing_plan_watch(state_manager, new_packing_plan):\n      Log.debug(\"State watch triggered for PackingPlan update on shard %s. Existing: %s, New: %s\" %\n                (self.shard, str(self.packing_plan), str(new_packing_plan)))\n\n      if self.packing_plan != new_packing_plan:\n        Log.info(\"PackingPlan change detected on shard %s, relaunching effected processes.\"\n                 % self.shard)\n        self.update_packing_plan(new_packing_plan)\n\n        Log.info(\"Updating executor processes\")\n        self.launch()\n      else:\n        Log.info(\n            \"State watch triggered for PackingPlan update but plan not changed so not relaunching.\")\n\n    for state_manager in self.state_managers:\n      # The callback function with the bound\n      # state_manager as first variable.\n      onPackingPlanWatch = functools.partial(on_packing_plan_watch, state_manager)\n      state_manager.get_packing_plan(self.topology_name, onPackingPlanWatch)\n      Log.info(\"Registered state watch for packing plan changes with state manager %s.\" %\n               str(state_manager))", "response": "Start watching for state managers and update processes as needed."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef run(self, name, config, builder):\n    if not isinstance(name, str):\n      raise RuntimeError(\"Name has to be a string type\")\n    if not isinstance(config, Config):\n      raise RuntimeError(\"config has to be a Config type\")\n    if not isinstance(builder, Builder):\n      raise RuntimeError(\"builder has to be a Builder type\")\n    bldr = TopologyBuilder(name=name)\n    builder.build(bldr)\n    bldr.set_config(config._api_config)\n    bldr.build_and_submit()", "response": "Builds the topology and submits it"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nforce every module in modList to be placed into main", "response": "def _modules_to_main(modList):\n  \"\"\"Force every module in modList to be placed into main\"\"\"\n  if not modList:\n    return\n\n  main = sys.modules['__main__']\n  for modname in modList:\n    if isinstance(modname, str):\n      try:\n        mod = __import__(modname)\n      except Exception:\n        sys.stderr.write(\n            'warning: could not import %s\\n.  '\n            'Your function may unexpectedly error due to this import failing;'\n            'A version mismatch is likely.  Specific error was:\\n' % modname)\n        print_exec(sys.stderr)\n      else:\n        setattr(main, mod.__name__, mod)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfills in the rest of the function data into the skeleton function object that were created via _make_skel_func.", "response": "def _fill_function(func, globalsn, defaults, dictn, module):\n  \"\"\" Fills in the rest of function data into the skeleton function object\n    that were created via _make_skel_func().\n     \"\"\"\n  func.__globals__.update(globalsn)\n  func.__defaults__ = defaults\n  func.__dict__ = dictn\n  func.__module__ = module\n\n  return func"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _make_skel_func(code, closures, base_globals=None):\n  closure = _reconstruct_closure(closures) if closures else None\n\n  if base_globals is None:\n    base_globals = {}\n  base_globals['__builtins__'] = __builtins__\n\n  return types.FunctionType(code, base_globals, None, None, closure)", "response": "Creates a skeleton function object that contains just the provided code and the correct number of cells in func_closure."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _load_class(cls, d):\n  for k, v in d.items():\n    if isinstance(k, tuple):\n      typ, k = k\n      if typ == 'property':\n        v = property(*v)\n      elif typ == 'staticmethod':\n        v = staticmethod(v) # pylint: disable=redefined-variable-type\n      elif typ == 'classmethod':\n        v = classmethod(v)\n    setattr(cls, k, v)\n  return cls", "response": "Load additional properties into class cls."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsave a module as an import", "response": "def save_module(self, obj):\n    \"\"\"\n    Save a module as an import\n    \"\"\"\n    self.modules.add(obj)\n    self.save_reduce(subimport, (obj.__name__,), obj=obj)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsaving a function object to the pickle file.", "response": "def save_function(self, obj, name=None):\n    \"\"\" Registered with the dispatch to handle all function types.\n    Determines what kind of function obj is (e.g. lambda, defined at\n    interactive prompt, etc) and handles the pickling appropriately.\n    \"\"\"\n    write = self.write\n\n    if name is None:\n      name = obj.__name__\n    try:\n      # whichmodule() could fail, see\n      # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling\n      modname = pickle.whichmodule(obj, name)\n    except Exception:\n      modname = None\n    # print('which gives %s %s %s' % (modname, obj, name))\n    try:\n      themodule = sys.modules[modname]\n    except KeyError:\n      # eval'd items such as namedtuple give invalid items for their function __module__\n      modname = '__main__'\n\n    if modname == '__main__':\n      themodule = None\n\n    if themodule:\n      self.modules.add(themodule)\n      if getattr(themodule, name, None) is obj:\n        return self.save_global(obj, name)\n\n    # if func is lambda, def'ed at prompt, is in main, or is nested, then\n    # we'll pickle the actual function object rather than simply saving a\n    # reference (as is done in default pickler), via save_function_tuple.\n    if islambda(obj) or obj.__code__.co_filename == '<stdin>' or themodule is None:\n      #print(\"save global\", islambda(obj), obj.__code__.co_filename, modname, themodule)\n      self.save_function_tuple(obj)\n      return\n    else:\n      # func is nested\n      klass = getattr(themodule, name, None)\n      if klass is None or klass is not obj:\n        self.save_function_tuple(obj)\n        return\n\n    if obj.__dict__:\n      # essentially save_reduce, but workaround needed to avoid recursion\n      self.save(_restore_attr)\n      write(pickle.MARK + pickle.GLOBAL + modname + '\\n' + name + '\\n')\n      self.memoize(obj)\n      self.save(obj.__dict__)\n      write(pickle.TUPLE + pickle.REDUCE)\n    else:\n      write(pickle.GLOBAL + modname + '\\n' + name + '\\n')\n      self.memoize(obj)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsaves a tuple of functions to disk.", "response": "def save_function_tuple(self, func):\n    \"\"\"  Pickles an actual func object.\n    A func comprises: code, globals, defaults, closure, and dict.  We\n    extract and save these, injecting reducing functions at certain points\n    to recreate the func object.  Keep in mind that some of these pieces\n    can contain a ref to the func itself.  Thus, a naive save on these\n    pieces could trigger an infinite loop of save's.  To get around that,\n    we first create a skeleton func object using just the code (this is\n    safe, since this won't contain a ref to the func), and memoize it as\n    soon as it's created.  The other stuff can then be filled in later.\n    \"\"\"\n    save = self.save\n    write = self.write\n\n    code, f_globals, defaults, closure, dct, base_globals = self.extract_func_data(func)\n\n    save(_fill_function)  # skeleton function updater\n    write(pickle.MARK)    # beginning of tuple that _fill_function expects\n\n    # create a skeleton function object and memoize it\n    save(_make_skel_func)\n    save((code, closure, base_globals))\n    write(pickle.REDUCE)\n    self.memoize(func)\n\n    # save the rest of the func data needed by _fill_function\n    save(f_globals)\n    save(defaults)\n    save(dct)\n    save(func.__module__)\n    write(pickle.TUPLE)\n    write(pickle.REDUCE)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsaves a file to the pickle file.", "response": "def save_file(self, obj): # pylint: disable=too-many-branches\n    \"\"\"Save a file\"\"\"\n    try:\n      import StringIO as pystringIO #we can't use cStringIO as it lacks the name attribute\n    except ImportError:\n      import io as pystringIO # pylint: disable=reimported\n\n    if not hasattr(obj, 'name') or  not hasattr(obj, 'mode'):\n      raise pickle.PicklingError(\"Cannot pickle files that do not map to an actual file\")\n    if obj is sys.stdout:\n      return self.save_reduce(getattr, (sys, 'stdout'), obj=obj)\n    if obj is sys.stderr:\n      return self.save_reduce(getattr, (sys, 'stderr'), obj=obj)\n    if obj is sys.stdin:\n      raise pickle.PicklingError(\"Cannot pickle standard input\")\n    if  hasattr(obj, 'isatty') and obj.isatty():\n      raise pickle.PicklingError(\"Cannot pickle files that map to tty objects\")\n    if 'r' not in obj.mode:\n      raise pickle.PicklingError(\"Cannot pickle files that are not opened for reading\")\n    name = obj.name\n    try:\n      fsize = os.stat(name).st_size\n    except OSError:\n      raise pickle.PicklingError(\"Cannot pickle file %s as it cannot be stat\" % name)\n\n    if obj.closed:\n      #create an empty closed string io\n      retval = pystringIO.StringIO(\"\")\n      retval.close()\n    elif not fsize: #empty file\n      retval = pystringIO.StringIO(\"\")\n      try:\n        tmpfile = file(name)\n        tst = tmpfile.read(1)\n      except IOError:\n        raise pickle.PicklingError(\"Cannot pickle file %s as it cannot be read\" % name)\n      tmpfile.close()\n      if tst != '':\n        raise pickle.PicklingError(\n            \"Cannot pickle file %s as it does not appear to map to a physical, real file\" % name)\n    else:\n      try:\n        tmpfile = file(name)\n        contents = tmpfile.read()\n        tmpfile.close()\n      except IOError:\n        raise pickle.PicklingError(\"Cannot pickle file %s as it cannot be read\" % name)\n      retval = pystringIO.StringIO(contents)\n      curloc = obj.tell()\n      retval.seek(curloc)\n\n    retval.name = name\n    self.save(retval)\n    self.memoize(obj)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef tail(filename, n):\n  size = os.path.getsize(filename)\n  with open(filename, \"rb\") as f:\n   fm = mmap.mmap(f.fileno(), 0, mmap.MAP_SHARED, mmap.PROT_READ)\n   try:\n      for i in xrange(size - 1, -1, -1):\n          if fm[i] == '\\n':\n             n -= 1\n             if n == -1:\n                break\n      return fm[i + 1 if i else 0:].splitlines()\n   finally:\n        fm.close()", "response": "Returns the last n lines from the filename. No exception handling."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a serializer for a given context", "response": "def get_serializer(context):\n    \"\"\"Returns a serializer for a given context\"\"\"\n    cluster_config = context.get_cluster_config()\n    serializer_clsname = cluster_config.get(constants.TOPOLOGY_SERIALIZER_CLASSNAME, None)\n    if serializer_clsname is None:\n      return PythonSerializer()\n    else:\n      try:\n        topo_pex_path = context.get_topology_pex_path()\n        pex_loader.load_pex(topo_pex_path)\n        serializer_cls = pex_loader.import_and_get_class(topo_pex_path, serializer_clsname)\n        serializer = serializer_cls()\n        return serializer\n      except Exception as e:\n        raise RuntimeError(\"Error with loading custom serializer class: %s, with error message: %s\"\n                           % (serializer_clsname, str(e)))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _run_once(self):\n    try:\n      self.do_wait()\n      self._execute_wakeup_tasks()\n      self._trigger_timers()\n    except Exception as e:\n      Log.error(\"Error occured during _run_once(): \" + str(e))\n      Log.error(traceback.format_exc())\n      self.should_exit = True", "response": "Run once in the loop."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nregister a task in the timer list", "response": "def register_timer_task_in_sec(self, task, second):\n    \"\"\"Registers a new timer task\n\n    :param task: function to be run at a specified second from now\n    :param second: how many seconds to wait before the timer is triggered\n    \"\"\"\n    # Python time is in float\n    second_in_float = float(second)\n    expiration = time.time() + second_in_float\n    heappush(self.timer_tasks, (expiration, task))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the next timeout interval from now This should be used from do_wait.", "response": "def _get_next_timeout_interval(self):\n    \"\"\"Get the next timeout from now\n\n    This should be used from do_wait().\n    :returns (float) next_timeout, or 10.0 if there are no timer events\n    \"\"\"\n    if len(self.timer_tasks) == 0:\n      return sys.maxsize\n    else:\n      next_timeout_interval = self.timer_tasks[0][0] - time.time()\n      return next_timeout_interval"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nexecute wakeup tasks in order to avoid concurrent calls to loop", "response": "def _execute_wakeup_tasks(self):\n    \"\"\"Executes wakeup tasks, should only be called from loop()\"\"\"\n    # Check the length of wakeup tasks first to avoid concurrent issues\n    size = len(self.wakeup_tasks)\n    for i in range(size):\n      self.wakeup_tasks[i]()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nposting processing of the process.", "response": "def post(self):\n    \"\"\" post method \"\"\"\n    def status_finish(ret):\n      self.set_status(ret)\n      self.finish()\n\n    def kill_parent():\n      status_finish(200)\n      logger.info(\"Killing parent executor\")\n      os.killpg(os.getppid(), signal.SIGTERM)\n\n    logger = logging.getLogger(__file__)\n    logger.info(\"Received 'Killing process' request\")\n    data = dict(urlparse.parse_qsl(self.request.body))\n\n    # check shared secret\n    sharedSecret = data.get('secret')\n    if sharedSecret != options.secret:\n      status_finish(403)\n      return\n\n    instanceId = data.get('instance_id_to_restart')\n    if instanceId:\n      filepath = instanceId + '.pid'\n      if os.path.isfile(filepath): # instance_id found\n        if instanceId.startswith('heron-executor-'): # kill heron-executor\n          kill_parent()\n        else: # kill other normal instance\n          fh = open(filepath)\n          firstLine = int(fh.readline())\n          fh.close()\n          logger.info(\"Killing process \" + instanceId + \" \" + str(firstLine))\n          os.kill(firstLine, signal.SIGTERM)\n          status_finish(200)\n      else: # instance_id not found\n        logger.info(filepath + \" not found\")\n        status_finish(422)\n    else: # instance_id not given, which means kill the container\n      kill_parent()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nexecute query with tmaster", "response": "def execute_query(self, tmaster, query_string, start, end):\n    \"\"\" execute query \"\"\"\n    if not tmaster:\n      raise Exception(\"No tmaster found\")\n    self.tmaster = tmaster\n    root = self.parse_query_string(query_string)\n    metrics = yield root.execute(self.tracker, self.tmaster, start, end)\n    raise tornado.gen.Return(metrics)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef find_closing_braces(self, query):\n    if query[0] != '(':\n      raise Exception(\"Trying to find closing braces for no opening braces\")\n    num_open_braces = 0\n    for i in range(len(query)):\n      c = query[i]\n      if c == '(':\n        num_open_braces += 1\n      elif c == ')':\n        num_open_braces -= 1\n      if num_open_braces == 0:\n        return i\n    raise Exception(\"No closing braces found\")", "response": "Find the index of the closing braces for the opening braces in the query string."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a parse tree for the query string.", "response": "def parse_query_string(self, query):\n    \"\"\"Returns a parse tree for the query, each of the node is a\n    subclass of Operator. This is both a lexical as well as syntax analyzer step.\"\"\"\n    if not query:\n      return None\n    # Just braces do not matter\n    if query[0] == '(':\n      index = self.find_closing_braces(query)\n      # This must be the last index, since this was an NOP starting brace\n      if index != len(query) - 1:\n        raise Exception(\"Invalid syntax\")\n      else:\n        return self.parse_query_string(query[1:-1])\n    start_index = query.find(\"(\")\n    # There must be a ( in the query\n    if start_index < 0:\n      # Otherwise it must be a constant\n      try:\n        constant = float(query)\n        return constant\n      except ValueError:\n        raise Exception(\"Invalid syntax\")\n    token = query[:start_index]\n    if token not in self.operators:\n      raise Exception(\"Invalid token: \" + token)\n\n    # Get sub components\n    rest_of_the_query = query[start_index:]\n    braces_end_index = self.find_closing_braces(rest_of_the_query)\n    if braces_end_index != len(rest_of_the_query) - 1:\n      raise Exception(\"Invalid syntax\")\n    parts = self.get_sub_parts(rest_of_the_query[1:-1])\n\n    # parts are simple strings in this case\n    if token == \"TS\":\n      # This will raise exception if parts are not syntactically correct\n      return self.operators[token](parts)\n\n    children = []\n    for part in parts:\n      children.append(self.parse_query_string(part))\n\n    # Make a node for the current token\n    node = self.operators[token](children)\n    return node"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets method executes the metrics query and writes the result to the client", "response": "def get(self):\n    \"\"\" get method \"\"\"\n    try:\n      cluster = self.get_argument_cluster()\n\n      role = self.get_argument_role()\n      environ = self.get_argument_environ()\n      topology_name = self.get_argument_topology()\n      topology = self.tracker.getTopologyByClusterRoleEnvironAndName(\n          cluster, role, environ, topology_name)\n\n      start_time = self.get_argument_starttime()\n      end_time = self.get_argument_endtime()\n      self.validateInterval(start_time, end_time)\n\n      query = self.get_argument_query()\n      metrics = yield tornado.gen.Task(self.executeMetricsQuery,\n                                       topology.tmaster, query, int(start_time), int(end_time))\n      self.write_success_response(metrics)\n    except Exception as e:\n      Log.debug(traceback.format_exc())\n      self.write_error_response(e)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nexecuting a query on the given tmaster and return the results as a list of dicts.", "response": "def executeMetricsQuery(self, tmaster, queryString, start_time, end_time, callback=None):\n    \"\"\"\n    Get the specified metrics for the given query in this topology.\n    Returns the following dict on success:\n    {\n      \"timeline\": [{\n        \"instance\": <instance>,\n        \"data\": {\n          <start_time> : <numeric value>,\n          <start_time> : <numeric value>,\n          ...\n        }\n      }, {\n        ...\n      }, ...\n      \"starttime\": <numeric value>,\n      \"endtime\": <numeric value>,\n    },\n\n    Returns the following dict on failure:\n    {\n      \"message\": \"...\"\n    }\n    \"\"\"\n\n    query = Query(self.tracker)\n    metrics = yield query.execute_query(tmaster, queryString, start_time, end_time)\n\n    # Parse the response\n    ret = {}\n    ret[\"starttime\"] = start_time\n    ret[\"endtime\"] = end_time\n    ret[\"timeline\"] = []\n\n    for metric in metrics:\n      tl = {\n          \"data\": metric.timeline\n      }\n      if metric.instance:\n        tl[\"instance\"] = metric.instance\n      ret[\"timeline\"].append(tl)\n\n    raise tornado.gen.Return(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a parser for the command.", "response": "def create_parser(subparsers):\n  '''\n  :param subparsers:\n  :return:\n  '''\n  parser = subparsers.add_parser(\n      'help',\n      help='Prints help for commands',\n      add_help=True)\n\n  # pylint: disable=protected-access\n  parser._positionals.title = \"Required arguments\"\n  parser._optionals.title = \"Optional arguments\"\n\n  parser.add_argument(\n      'help-command',\n      nargs='?',\n      default='help',\n      help='Provide help for a command')\n\n  parser.set_defaults(subcommand='help')\n  return parser"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef emit(self, tup, stream=Stream.DEFAULT_STREAM_ID,\n           anchors=None, direct_task=None, need_task_ids=False):\n    \"\"\"Emits a new tuple from this Bolt\n\n    It is compatible with StreamParse API.\n\n    :type tup: list or tuple\n    :param tup: the new output Tuple to send from this bolt,\n                should only contain only serializable data.\n    :type stream: str\n    :param stream: the ID of the stream to emit this Tuple to.\n                   Leave empty to emit to the default stream.\n    :type anchors: list\n    :param anchors: a list of HeronTuples to which the emitted Tuples should be anchored.\n    :type direct_task: int\n    :param direct_task: the task to send the Tupel to if performing a direct emit.\n    :type need_task_ids: bool\n    :param need_task_ids: indicate whether or not you would like the task IDs the Tuple was emitted.\n    \"\"\"\n    # first check whether this tuple is sane\n    self.pplan_helper.check_output_schema(stream, tup)\n\n    # get custom grouping target task ids; get empty list if not custom grouping\n    custom_target_task_ids = self.pplan_helper.choose_tasks_for_custom_grouping(stream, tup)\n\n    self.pplan_helper.context.invoke_hook_emit(tup, stream, None)\n\n    data_tuple = tuple_pb2.HeronDataTuple()\n    data_tuple.key = 0\n\n    if direct_task is not None:\n      if not isinstance(direct_task, int):\n        raise TypeError(\"direct_task argument needs to be an integer, given: %s\"\n                        % str(type(direct_task)))\n      # performing emit-direct\n      data_tuple.dest_task_ids.append(direct_task)\n    elif custom_target_task_ids is not None:\n      for task_id in custom_target_task_ids:\n        # for custom grouping\n        data_tuple.dest_task_ids.append(task_id)\n\n    # Set the anchors for a tuple\n    if anchors is not None:\n      merged_roots = set()\n      for tup in [t for t in anchors if isinstance(t, HeronTuple) and t.roots is not None]:\n        merged_roots.update(tup.roots)\n      for rt in merged_roots:\n        to_add = data_tuple.roots.add()\n        to_add.CopyFrom(rt)\n\n    tuple_size_in_bytes = 0\n    start_time = time.time()\n\n    # Serialize\n    for obj in tup:\n      serialized = self.serializer.serialize(obj)\n      data_tuple.values.append(serialized)\n      tuple_size_in_bytes += len(serialized)\n    serialize_latency_ns = (time.time() - start_time) * system_constants.SEC_TO_NS\n    self.bolt_metrics.serialize_data_tuple(stream, serialize_latency_ns)\n\n    super(BoltInstance, self).admit_data_tuple(stream_id=stream, data_tuple=data_tuple,\n                                               tuple_size_in_bytes=tuple_size_in_bytes)\n\n    self.bolt_metrics.update_emit_count(stream)\n    if need_task_ids:\n      sent_task_ids = custom_target_task_ids or []\n      if direct_task is not None:\n        sent_task_ids.append(direct_task)\n      return sent_task_ids", "response": "Emits a new tuple from this Bolt\n   "}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncall when tuple was buffered into in_stream", "response": "def process_incoming_tuples(self):\n    \"\"\"Should be called when tuple was buffered into in_stream\n\n    This method is equivalent to ``addBoltTasks()`` but\n    is designed for event-driven single-thread bolt.\n    \"\"\"\n    # back-pressure\n    if self.output_helper.is_out_queue_available():\n      self._read_tuples_and_execute()\n      self.output_helper.send_out_tuples()\n    else:\n      # update outqueue full count\n      self.bolt_metrics.update_out_queue_full_count()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nindicating that processing of a tuple has succeeded", "response": "def ack(self, tup):\n    \"\"\"Indicate that processing of a Tuple has succeeded\n\n    It is compatible with StreamParse API.\n    \"\"\"\n    if not isinstance(tup, HeronTuple):\n      Log.error(\"Only HeronTuple type is supported in ack()\")\n      return\n\n    if self.acking_enabled:\n      ack_tuple = tuple_pb2.AckTuple()\n      ack_tuple.ackedtuple = int(tup.id)\n\n      tuple_size_in_bytes = 0\n      for rt in tup.roots:\n        to_add = ack_tuple.roots.add()\n        to_add.CopyFrom(rt)\n        tuple_size_in_bytes += rt.ByteSize()\n      super(BoltInstance, self).admit_control_tuple(ack_tuple, tuple_size_in_bytes, True)\n\n    process_latency_ns = (time.time() - tup.creation_time) * system_constants.SEC_TO_NS\n    self.pplan_helper.context.invoke_hook_bolt_ack(tup, process_latency_ns)\n    self.bolt_metrics.acked_tuple(tup.stream, tup.component, process_latency_ns)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nindicate that processing of a tuple has failed", "response": "def fail(self, tup):\n    \"\"\"Indicate that processing of a Tuple has failed\n\n    It is compatible with StreamParse API.\n    \"\"\"\n    if not isinstance(tup, HeronTuple):\n      Log.error(\"Only HeronTuple type is supported in fail()\")\n      return\n\n    if self.acking_enabled:\n      fail_tuple = tuple_pb2.AckTuple()\n      fail_tuple.ackedtuple = int(tup.id)\n\n      tuple_size_in_bytes = 0\n      for rt in tup.roots:\n        to_add = fail_tuple.roots.add()\n        to_add.CopyFrom(rt)\n        tuple_size_in_bytes += rt.ByteSize()\n      super(BoltInstance, self).admit_control_tuple(fail_tuple, tuple_size_in_bytes, False)\n\n    fail_latency_ns = (time.time() - tup.creation_time) * system_constants.SEC_TO_NS\n    self.pplan_helper.context.invoke_hook_bolt_fail(tup, fail_latency_ns)\n    self.bolt_metrics.failed_tuple(tup.stream, tup.component, fail_latency_ns)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef execute(handlers):\n  '''\n  Run the command\n  :return:\n  '''\n  # verify if the environment variables are correctly set\n  check_environment()\n\n  # create the argument parser\n  parser = create_parser(handlers)\n\n  # if no argument is provided, print help and exit\n  if len(sys.argv[1:]) == 0:\n    parser.print_help()\n    return 0\n\n  # insert the boolean values for some of the options\n  sys.argv = config.insert_bool_values(sys.argv)\n\n  try:\n    # parse the args\n    args, unknown_args = parser.parse_known_args()\n  except ValueError as ex:\n    Log.error(\"Error while parsing arguments: %s\", str(ex))\n    Log.debug(traceback.format_exc())\n    sys.exit(1)\n\n  command_line_args = vars(args)\n\n  # set log level\n  log.set_logging_level(command_line_args)\n  Log.debug(\"Input Command Line Args: %s\", command_line_args)\n\n  # command to be execute\n  command = command_line_args['subcommand']\n\n  # print the input parameters, if verbose is enabled\n  Log.debug(\"Processed Command Line Args: %s\", command_line_args)\n\n  results = run(handlers, command, parser, command_line_args, unknown_args)\n\n  return 0 if result.is_successful(results) else 1", "response": "Runs the command\n Execute the command\n "}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a subparser for the standalone command", "response": "def create_parser(subparsers):\n  '''\n  Create a subparser for the standalone command\n  :param subparsers:\n  :return:\n  '''\n  parser = subparsers.add_parser(\n      'standalone',\n      help='Start a standalone Heron cluster',\n      add_help=True\n  )\n\n  cli_args.add_titles(parser)\n\n  parser_action = parser.add_subparsers()\n\n  parser_cluster = parser_action.add_parser(\n      Action.CLUSTER,\n      help='Start or stop cluster',\n      add_help=True,\n      formatter_class=argparse.RawTextHelpFormatter,\n  )\n  parser_cluster.set_defaults(action=Action.CLUSTER)\n\n  parser_set = parser_action.add_parser(\n      Action.SET,\n      help='Set configurations for standalone cluster e.g. master or slave nodes',\n      add_help=True,\n      formatter_class=argparse.RawTextHelpFormatter\n  )\n  parser_set.set_defaults(action=Action.SET)\n\n  parser_template = parser_action.add_parser(\n      Action.TEMPLATE,\n      help='Template Heron configurations based on cluster roles',\n      add_help=True,\n      formatter_class=argparse.RawTextHelpFormatter\n  )\n  parser_template.set_defaults(action=Action.TEMPLATE)\n\n  parser_cluster.add_argument(\n      TYPE,\n      type=str,\n      choices={Cluster.START, Cluster.STOP},\n      help= \\\n\"\"\"\nChoices supports the following:\n  start     - Start standalone Heron cluster\n  stop      - Stop standalone Heron cluster\n\"\"\"\n  )\n\n  parser_template.add_argument(\n      TYPE,\n      type=str,\n      choices={\"configs\"},\n  )\n\n  parser_get = parser_action.add_parser(\n      Action.GET,\n      help='Get attributes about the standalone cluster',\n      add_help=True,\n      formatter_class=argparse.RawTextHelpFormatter\n  )\n  parser_get.set_defaults(action=Action.GET)\n\n  parser_get.add_argument(\n      TYPE,\n      type=str,\n      choices={Get.SERVICE_URL, Get.HERON_TRACKER_URL, Get.HERON_UI_URL},\n      help= \\\n      \"\"\"\n      Choices supports the following:\n        service-url         - Get the service url for standalone cluster\n        heron-tracker-url   - Get the url for the heron tracker in standalone cluster\n        heron-ui-url        - Get the url for the heron ui standalone cluster\n      \"\"\"\n  )\n\n  parser_info = parser_action.add_parser(\n      Action.INFO,\n      help='Get general information about the standalone cluster',\n      add_help=True,\n      formatter_class=argparse.RawTextHelpFormatter\n  )\n  parser_info.set_defaults(action=Action.INFO)\n\n  add_additional_args([parser_set, parser_cluster, parser_template, parser_get, parser_info])\n  parser.set_defaults(subcommand='standalone')\n  return parser"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ntemplate slave config file", "response": "def template_slave_hcl(cl_args, masters):\n  '''\n  Template slave config file\n  '''\n  slave_config_template = \"%s/standalone/templates/slave.template.hcl\" % cl_args[\"config_path\"]\n  slave_config_actual = \"%s/standalone/resources/slave.hcl\" % cl_args[\"config_path\"]\n  masters_in_quotes = ['\"%s\"' % master for master in masters]\n  template_file(slave_config_template, slave_config_actual,\n                {\"<nomad_masters:master_port>\": \", \".join(masters_in_quotes)})"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntemplates the apiserver. hcl file for the given master and zookeepers.", "response": "def template_apiserver_hcl(cl_args, masters, zookeepers):\n  \"\"\"\n  template apiserver.hcl\n  \"\"\"\n  single_master = masters[0]\n  apiserver_config_template = \"%s/standalone/templates/apiserver.template.hcl\" \\\n                              % cl_args[\"config_path\"]\n  apiserver_config_actual = \"%s/standalone/resources/apiserver.hcl\" % cl_args[\"config_path\"]\n\n  replacements = {\n      \"<heron_apiserver_hostname>\": '\"%s\"' % get_hostname(single_master, cl_args),\n      \"<heron_apiserver_executable>\": '\"%s/heron-apiserver\"'\n                                      % config.get_heron_bin_dir()\n                                      if is_self(single_master)\n                                      else '\"%s/.heron/bin/heron-apiserver\"'\n                                      % get_remote_home(single_master, cl_args),\n      \"<zookeeper_host:zookeeper_port>\": \",\".join(\n          ['%s' % zk if \":\" in zk else '%s:2181' % zk for zk in zookeepers]),\n      \"<scheduler_uri>\": \"http://%s:4646\" % single_master\n  }\n\n  template_file(apiserver_config_template, apiserver_config_actual, replacements)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef print_cluster_info(cl_args):\n  '''\n  get cluster info for standalone cluster\n  '''\n  parsed_roles = read_and_parse_roles(cl_args)\n  masters = list(parsed_roles[Role.MASTERS])\n  slaves = list(parsed_roles[Role.SLAVES])\n  zookeepers = list(parsed_roles[Role.ZOOKEEPERS])\n  cluster = list(parsed_roles[Role.CLUSTER])\n\n  # OrderedDicts are used here so that the key order can be\n  # specified directly\n  info = OrderedDict()\n  info['numNodes'] = len(cluster)\n  info['nodes'] = cluster\n  roles = OrderedDict()\n  roles['masters'] = masters\n  roles['slaves'] = slaves\n  roles['zookeepers'] = zookeepers\n  urls = OrderedDict()\n  urls['serviceUrl'] = get_service_url(cl_args)\n  urls['heronUi'] = get_heron_ui_url(cl_args)\n  urls['heronTracker'] = get_heron_tracker_url(cl_args)\n  info['roles'] = roles\n  info['urls'] = urls\n\n  print json.dumps(info, indent=2)", "response": "get cluster info for standalone cluster\n \u2192 print info about the cluster"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef add_additional_args(parsers):\n  '''\n  add additional parameters to parser\n  '''\n  for parser in parsers:\n    cli_args.add_verbose(parser)\n    cli_args.add_config(parser)\n    parser.add_argument(\n        '--heron-dir',\n        default=config.get_heron_dir(),\n        help='Path to Heron home directory')", "response": "add additional parameters to parser\n"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nstarts a Heron standalone cluster.", "response": "def start_cluster(cl_args):\n  '''\n  Start a Heron standalone cluster\n  '''\n  roles = read_and_parse_roles(cl_args)\n  masters = roles[Role.MASTERS]\n  slaves = roles[Role.SLAVES]\n  zookeepers = roles[Role.ZOOKEEPERS]\n  Log.info(\"Roles:\")\n  Log.info(\" - Master Servers: %s\" % list(masters))\n  Log.info(\" - Slave Servers: %s\" % list(slaves))\n  Log.info(\" - Zookeeper Servers: %s\" % list(zookeepers))\n  if not masters:\n    Log.error(\"No master servers specified!\")\n    sys.exit(-1)\n  if not slaves:\n    Log.error(\"No slave servers specified!\")\n    sys.exit(-1)\n  if not zookeepers:\n    Log.error(\"No zookeeper servers specified!\")\n    sys.exit(-1)\n  # make sure configs are templated\n  update_config_files(cl_args)\n\n  dist_nodes = list(masters.union(slaves))\n  # if just local deployment\n  if not (len(dist_nodes) == 1 and is_self(dist_nodes[0])):\n    distribute_package(roles, cl_args)\n  start_master_nodes(masters, cl_args)\n  start_slave_nodes(slaves, cl_args)\n  start_api_server(masters, cl_args)\n  start_heron_tools(masters, cl_args)\n  Log.info(\"Heron standalone cluster complete!\")"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef start_heron_tools(masters, cl_args):\n  '''\n  Start Heron tracker and UI\n  '''\n  single_master = list(masters)[0]\n  wait_for_master_to_start(single_master)\n\n  cmd = \"%s run %s >> /tmp/heron_tools_start.log 2>&1 &\" \\\n        % (get_nomad_path(cl_args), get_heron_tools_job_file(cl_args))\n  Log.info(\"Starting Heron Tools on %s\" % single_master)\n\n  if not is_self(single_master):\n    cmd = ssh_remote_execute(cmd, single_master, cl_args)\n  Log.debug(cmd)\n  pid = subprocess.Popen(cmd,\n                         shell=True,\n                         stdout=subprocess.PIPE,\n                         stderr=subprocess.PIPE)\n\n  return_code = pid.wait()\n  output = pid.communicate()\n  Log.debug(\"return code: %s output: %s\" % (return_code, output))\n  if return_code != 0:\n    Log.error(\"Failed to start Heron Tools on %s with error:\\n%s\" % (single_master, output[1]))\n    sys.exit(-1)\n\n  wait_for_job_to_start(single_master, \"heron-tools\")\n  Log.info(\"Done starting Heron Tools\")", "response": "Start Heron Tools on a single master."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndistribute Heron packages to all nodes", "response": "def distribute_package(roles, cl_args):\n  '''\n  distribute Heron packages to all nodes\n  '''\n  Log.info(\"Distributing heron package to nodes (this might take a while)...\")\n  masters = roles[Role.MASTERS]\n  slaves = roles[Role.SLAVES]\n\n  tar_file = tempfile.NamedTemporaryFile(suffix=\".tmp\").name\n  Log.debug(\"TAR file %s to %s\" % (cl_args[\"heron_dir\"], tar_file))\n  make_tarfile(tar_file, cl_args[\"heron_dir\"])\n  dist_nodes = masters.union(slaves)\n\n  scp_package(tar_file, dist_nodes, cl_args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwait for a nomad master to start", "response": "def wait_for_master_to_start(single_master):\n  '''\n  Wait for a nomad master to start\n  '''\n  i = 0\n  while True:\n    try:\n      r = requests.get(\"http://%s:4646/v1/status/leader\" % single_master)\n      if r.status_code == 200:\n        break\n    except:\n      Log.debug(sys.exc_info()[0])\n      Log.info(\"Waiting for cluster to come up... %s\" % i)\n      time.sleep(1)\n      if i > 10:\n        Log.error(\"Failed to start Nomad Cluster!\")\n        sys.exit(-1)\n    i = i + 1"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwait for a Nomad job to start", "response": "def wait_for_job_to_start(single_master, job):\n  '''\n  Wait for a Nomad job to start\n  '''\n  i = 0\n  while True:\n    try:\n      r = requests.get(\"http://%s:4646/v1/job/%s\" % (single_master, job))\n      if r.status_code == 200 and r.json()[\"Status\"] == \"running\":\n        break\n      else:\n        raise RuntimeError()\n    except:\n      Log.debug(sys.exc_info()[0])\n      Log.info(\"Waiting for %s to come up... %s\" % (job, i))\n      time.sleep(1)\n      if i > 20:\n        Log.error(\"Failed to start Nomad Cluster!\")\n        sys.exit(-1)\n    i = i + 1"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef scp_package(package_file, destinations, cl_args):\n  '''\n  scp and extract package\n  '''\n  pids = []\n  for dest in destinations:\n    if is_self(dest):\n      continue\n    Log.info(\"Server: %s\" % dest)\n    file_path = \"/tmp/heron.tar.gz\"\n    dest_file_path = \"%s:%s\" % (dest, file_path)\n\n    remote_cmd = \"rm -rf ~/.heron && mkdir ~/.heron \" \\\n                 \"&& tar -xzvf %s -C ~/.heron --strip-components 1\" % (file_path)\n    cmd = '%s && %s' \\\n          % (scp_cmd(package_file, dest_file_path, cl_args),\n             ssh_remote_execute(remote_cmd, dest, cl_args))\n    Log.debug(cmd)\n    pid = subprocess.Popen(cmd,\n                           shell=True,\n                           stdout=subprocess.PIPE,\n                           stderr=subprocess.PIPE)\n    pids.append({\"pid\": pid, \"dest\": dest})\n\n  errors = []\n  for entry in pids:\n    pid = entry[\"pid\"]\n    return_code = pid.wait()\n    output = pid.communicate()\n    Log.debug(\"return code: %s output: %s\" % (return_code, output))\n    if return_code != 0:\n      errors.append(\"Failed to scp package to %s with error:\\n%s\" % (entry[\"dest\"], output[1]))\n\n  if errors:\n    for error in errors:\n      Log.error(error)\n    sys.exit(-1)\n\n  Log.info(\"Done distributing packages\")", "response": "scp and extract package\n \u00b0"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef make_tarfile(output_filename, source_dir):\n  '''\n  Tar a directory\n  '''\n  with tarfile.open(output_filename, \"w:gz\") as tar:\n    tar.add(source_dir, arcname=os.path.basename(source_dir))", "response": "Create a tar file from a source directory."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read_and_parse_roles(cl_args):\n  '''\n  read config files to get roles\n  '''\n  roles = dict()\n\n  with open(get_inventory_file(cl_args), 'r') as stream:\n    try:\n      roles = yaml.load(stream)\n    except yaml.YAMLError as exc:\n      Log.error(\"Error parsing inventory file: %s\" % exc)\n      sys.exit(-1)\n\n  if Role.ZOOKEEPERS not in roles or not roles[Role.ZOOKEEPERS]:\n    Log.error(\"Zookeeper servers node defined!\")\n    sys.exit(-1)\n\n  if Role.CLUSTER not in roles or not roles[Role.CLUSTER]:\n    Log.error(\"Heron cluster nodes defined!\")\n    sys.exit(-1)\n\n  # Set roles\n  roles[Role.MASTERS] = set([roles[Role.CLUSTER][0]])\n  roles[Role.SLAVES] = set(roles[Role.CLUSTER])\n  roles[Role.ZOOKEEPERS] = set(roles[Role.ZOOKEEPERS])\n  roles[Role.CLUSTER] = set(roles[Role.CLUSTER])\n\n  return roles", "response": "read config files to get roles\n \u2013 parse them and return a dict"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting remote home directory of remote host", "response": "def get_remote_home(host, cl_args):\n  '''\n  get home directory of remote host\n  '''\n  cmd = \"echo ~\"\n  if not is_self(host):\n    cmd = ssh_remote_execute(cmd, host, cl_args)\n  pid = subprocess.Popen(cmd,\n                         shell=True,\n                         stdout=subprocess.PIPE,\n                         stderr=subprocess.PIPE)\n  return_code = pid.wait()\n  output = pid.communicate()\n\n  if return_code != 0:\n    Log.error(\"Failed to get home path for remote host %s with output:\\n%s\" % (host, output))\n    sys.exit(-1)\n  return output[0].strip(\"\\n\")"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget host name of remote host", "response": "def get_hostname(ip_addr, cl_args):\n  '''\n  get host name of remote host\n  '''\n  if is_self(ip_addr):\n    return get_self_hostname()\n  cmd = \"hostname\"\n  ssh_cmd = ssh_remote_execute(cmd, ip_addr, cl_args)\n  pid = subprocess.Popen(ssh_cmd,\n                         shell=True,\n                         stdout=subprocess.PIPE,\n                         stderr=subprocess.PIPE)\n  return_code = pid.wait()\n  output = pid.communicate()\n\n  if return_code != 0:\n    Log.error(\"Failed to get hostname for remote host %s with output:\\n%s\" % (ip_addr, output))\n    sys.exit(-1)\n  return output[0].strip(\"\\n\")"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncheck if this host is this addr check if this host is self addr check if this host is self addr athor", "response": "def is_self(addr):\n  '''\n  check if this host is this addr\n  '''\n  ips = []\n  for i in netifaces.interfaces():\n    entry = netifaces.ifaddresses(i)\n    if netifaces.AF_INET in entry:\n      for ipv4 in entry[netifaces.AF_INET]:\n        if \"addr\" in ipv4:\n          ips.append(ipv4[\"addr\"])\n  return addr in ips or addr == get_self_hostname()"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef log(self, message, level=None):\n    if level is None:\n      _log_level = logging.INFO\n    else:\n      if level == \"trace\" or level == \"debug\":\n        _log_level = logging.DEBUG\n      elif level == \"info\":\n        _log_level = logging.INFO\n      elif level == \"warn\":\n        _log_level = logging.WARNING\n      elif level == \"error\":\n        _log_level = logging.ERROR\n      else:\n        raise ValueError(\"%s is not supported as logging level\" % str(level))\n\n    self.logger.log(_log_level, message)", "response": "Log a message to the log file."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load_py_instance(self, is_spout):\n    try:\n      if is_spout:\n        spout_proto = self.pplan_helper.get_my_spout()\n        py_classpath = spout_proto.comp.class_name\n        self.logger.info(\"Loading Spout from: %s\", py_classpath)\n      else:\n        bolt_proto = self.pplan_helper.get_my_bolt()\n        py_classpath = bolt_proto.comp.class_name\n        self.logger.info(\"Loading Bolt from: %s\", py_classpath)\n\n      pex_loader.load_pex(self.pplan_helper.topology_pex_abs_path)\n      spbl_class = pex_loader.import_and_get_class(self.pplan_helper.topology_pex_abs_path,\n                                                   py_classpath)\n    except Exception as e:\n      spbl = \"spout\" if is_spout else \"bolt\"\n      self.logger.error(traceback.format_exc())\n      raise RuntimeError(\"Error when loading a %s from pex: %s\" % (spbl, str(e)))\n    return spbl_class", "response": "Loads a user defined component from the pex file."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets all the values for the cluster and environ.", "response": "def get(self):\n    \"\"\" get method \"\"\"\n    # Get all the values for parameter \"cluster\".\n    clusters = self.get_arguments(constants.PARAM_CLUSTER)\n    # Get all the values for parameter \"environ\".\n    environs = self.get_arguments(constants.PARAM_ENVIRON)\n    # Get role\n    role = self.get_argument_role()\n\n    ret = {}\n    topologies = self.tracker.topologies\n    for topology in topologies:\n      cluster = topology.cluster\n      environ = topology.environ\n      execution_state = topology.execution_state\n\n      if not cluster or not execution_state or not environ:\n        continue\n\n      topo_role = execution_state.role\n      if not topo_role:\n        continue\n\n      # This cluster is not asked for.\n      # Note that \"if not clusters\", then\n      # we show for all the clusters.\n      if clusters and cluster not in clusters:\n        continue\n\n      # This environ is not asked for.\n      # Note that \"if not environs\", then\n      # we show for all the environs.\n      if environs and environ not in environs:\n        continue\n\n      # This role is not asked for.\n      # Note that \"if not role\", then\n      # we show for all the roles.\n      if role and role != topo_role:\n        continue\n\n      if cluster not in ret:\n        ret[cluster] = {}\n      if topo_role not in ret[cluster]:\n        ret[cluster][topo_role] = {}\n      if environ not in ret[cluster][topo_role]:\n        ret[cluster][topo_role][environ] = []\n      ret[cluster][topo_role][environ].append(topology.name)\n    self.write_success_response(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nresolving all symbolic references that src points to.", "response": "def dereference_symlinks(src):\n    \"\"\"\n    Resolve all symbolic references that `src` points to.  Note that this\n    is different than `os.path.realpath` as path components leading up to\n    the final location may still be symbolic links.\n    \"\"\"\n    while os.path.islink(src):\n        src = os.path.join(os.path.dirname(src), os.readlink(src))\n\n    return src"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get(self):\n    clusters = [statemgr.name for statemgr in self.tracker.state_managers]\n\n    self.write_success_response(clusters)", "response": "get method returns list of all available state managers"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting the list of topologies given a cluster submitted by a given role under a given environment", "response": "def get_cluster_role_env_topologies(cluster, role, env):\n  '''\n  Get the list of topologies given a cluster submitted by a given role under a given environment\n  :param cluster:\n  :param role:\n  :param env:\n  :return:\n  '''\n  return _get_topologies(cluster, role=role, env=env)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nget the execution state of a topology in a cluster.", "response": "def get_execution_state(cluster, environ, topology, role=None):\n  '''\n  Get the execution state of a topology in a cluster\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param role:\n  :return:\n  '''\n  params = dict(cluster=cluster, environ=environ, topology=topology)\n  if role is not None:\n    params['role'] = role\n  request_url = tornado.httputil.url_concat(create_url(EXECUTION_STATE_URL_FMT), params)\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the logical plan state of a topology in a cluster.", "response": "def get_logical_plan(cluster, environ, topology, role=None):\n  '''\n  Get the logical plan state of a topology in a cluster\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param role:\n  :return:\n  '''\n  params = dict(cluster=cluster, environ=environ, topology=topology)\n  if role is not None:\n    params['role'] = role\n  request_url = tornado.httputil.url_concat(\n      create_url(LOGICALPLAN_URL_FMT), params)\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the list of component names for the topology from Heron Nest", "response": "def get_comps(cluster, environ, topology, role=None):\n  '''\n  Get the list of component names for the topology from Heron Nest\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param role:\n  :return:\n  '''\n  params = dict(cluster=cluster, environ=environ, topology=topology)\n  if role is not None:\n    params['role'] = role\n  request_url = tornado.httputil.url_concat(\n      create_url(LOGICALPLAN_URL_FMT), params)\n  lplan = yield fetch_url_as_json(request_url)\n  comps = lplan['spouts'].keys() + lplan['bolts'].keys()\n  raise tornado.gen.Return(comps)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nget the list of instances for the topology from Heron Nest", "response": "def get_instances(cluster, environ, topology, role=None):\n  '''\n  Get the list of instances for the topology from Heron Nest\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param role:\n  :return:\n  '''\n  params = dict(cluster=cluster, environ=environ, topology=topology)\n  if role is not None:\n    params['role'] = role\n  request_url = tornado.httputil.url_concat(\n      create_url(PHYSICALPLAN_URL_FMT), params)\n  pplan = yield fetch_url_as_json(request_url)\n  instances = pplan['instances'].keys()\n  raise tornado.gen.Return(instances)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_physical_plan(cluster, environ, topology, role=None):\n  '''\n  Get the physical plan state of a topology in a cluster from tracker\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param role:\n  :return:\n  '''\n  params = dict(cluster=cluster, environ=environ, topology=topology)\n  if role is not None:\n    params['role'] = role\n  request_url = tornado.httputil.url_concat(\n      create_url(PHYSICALPLAN_URL_FMT), params)\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))", "response": "Get the physical plan state of a topology in a cluster from tracker\n"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting the scheduler location of a topology in a cluster from tracker", "response": "def get_scheduler_location(cluster, environ, topology, role=None):\n  '''\n  Get the scheduler location of a topology in a cluster from tracker\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param role:\n  :return:\n  '''\n  params = dict(cluster=cluster, environ=environ, topology=topology)\n  if role is not None:\n    params['role'] = role\n  request_url = tornado.httputil.url_concat(\n      create_url(SCHEDULER_LOCATION_URL_FMT), params)\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets summary of exception for a component.", "response": "def get_component_exceptionsummary(cluster, environ, topology, component, role=None):\n  '''\n  Get summary of exception for a component\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param component:\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      component=component)\n  if role is not None:\n    params['role'] = role\n  request_url = tornado.httputil.url_concat(\n      create_url(EXCEPTION_SUMMARY_URL_FMT), params)\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_component_exceptions(cluster, environ, topology, component, role=None):\n  '''\n  Get exceptions for 'component' for 'topology'\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param component:\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      component=component)\n  if role is not None:\n    params['role'] = role\n  request_url = tornado.httputil.url_concat(\n      create_url(EXCEPTIONS_URL_FMT), params)\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))", "response": "Get exceptions for component for cluster."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_comp_instance_metrics(cluster, environ, topology, component,\n                              metrics, instances, time_range, role=None):\n  '''\n  Get the metrics for some instances of a topology from tracker\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param component:\n  :param metrics:     dict of display name to cuckoo name\n  :param instances:\n  :param time_range:  2-tuple consisting of start and end of range\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      component=component)\n  if role is not None:\n    params['role'] = role\n\n  # form the fetch url\n  request_url = tornado.httputil.url_concat(\n      create_url(METRICS_URL_FMT), params)\n\n  # convert a single instance to a list, if needed\n  all_instances = instances if isinstance(instances, list) else [instances]\n\n  # append each metric to the url\n  for _, metric_name in metrics.items():\n    request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name[0]))\n\n  # append each instance to the url\n  for i in all_instances:\n    request_url = tornado.httputil.url_concat(request_url, dict(instance=i))\n\n  # append the time interval to the url\n  request_url = tornado.httputil.url_concat(request_url, dict(interval=time_range[1]))\n\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))", "response": "Get the metrics for some instances of a topology from tracker\n "}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets the metrics for all the instances of a topology from Heron Nest", "response": "def get_comp_metrics(cluster, environ, topology, component,\n                     instances, metricnames, time_range, role=None):\n  '''\n  Get the metrics for all the instances of a topology from Heron Nest\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param component:\n  :param instances:\n  :param metricnames:   dict of display name to cuckoo name\n  :param time_range:    2-tuple consisting of start and end of range\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      component=component)\n  if role is not None:\n    params['role'] = role\n\n  # form the url\n  request_url = tornado.httputil.url_concat(\n      create_url(METRICS_URL_FMT), params)\n\n  # append each metric to the url\n  for metric_name in metricnames:\n    request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name))\n\n  # append each instance to the url\n  for instance in instances:\n    request_url = tornado.httputil.url_concat(request_url, dict(instance=instance))\n\n  # append the time interval to the url\n  request_url = tornado.httputil.url_concat(request_url, dict(interval=time_range[1]))\n\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_metrics(cluster, environment, topology, timerange, query, role=None):\n  '''\n  Get the metrics for a topology from tracker\n  :param cluster:\n  :param environment:\n  :param topology:\n  :param timerange:\n  :param query:\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environment,\n      topology=topology,\n      starttime=timerange[0],\n      endtime=timerange[1],\n      query=query)\n\n  if role is not None:\n    params['role'] = role\n\n  request_url = tornado.httputil.url_concat(\n      create_url(METRICS_QUERY_URL_FMT), params\n  )\n\n  logging.info(\"get_metrics %s\", request_url)\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))", "response": "Get the metrics for a topology from tracker\n"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the minute - by - minute metrics for all instances of a topology from tracker This function returns the minute - by - minute metrics for all instances of a topology from tracker", "response": "def get_comp_metrics_timeline(cluster, environ, topology, component,\n                              instances, metricnames, time_range, role=None):\n  '''\n  Get the minute-by-minute metrics for all instances of a topology from tracker\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param component:\n  :param instances:\n  :param metricnames:   dict of display name to cuckoo name\n  :param time_range:    2-tuple consisting of start and end of range\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      component=component)\n\n  if role is not None:\n    params['role'] = role\n\n  # form the url\n  request_url = tornado.httputil.url_concat(create_url(METRICS_TIMELINE_URL_FMT), params)\n\n  if role is not None:\n    request_url = tornado.httputil.url_concat(request_url, dict(role=role))\n\n  # append each metric to the url\n  for metric_name in metricnames:\n    request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name))\n\n  # append each instance to the url\n  for instance in instances:\n    request_url = tornado.httputil.url_concat(request_url, dict(instance=instance))\n\n  # append the time interval to the url\n  request_url = tornado.httputil.url_concat(\n      request_url, dict(starttime=time_range[0], endtime=time_range[1]))\n\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_topology_info(cluster, environ, topology, role=None):\n  '''\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology)\n\n  if role is not None:\n    params['role'] = role\n\n  request_url = tornado.httputil.url_concat(create_url(INFO_URL_FMT), params)\n\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))", "response": "Get topology info from the server."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_instance_pid(cluster, environ, topology, instance, role=None):\n  '''\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param instance:\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      instance=instance)\n\n  if role is not None:\n    params['role'] = role\n\n  request_url = tornado.httputil.url_concat(create_url(PID_URL_FMT), params)\n\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))", "response": "Get the PID of the instance."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_instance_jstack(cluster, environ, topology, instance, role=None):\n  '''\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param instance:\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      instance=instance)\n\n  if role is not None:\n    params['role'] = role\n\n  request_url = tornado.httputil.url_concat(\n      create_url(JSTACK_URL_FMT), params)\n\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))", "response": "Get the JStack for a single instance."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the memory histogram of an instance.", "response": "def get_instance_mem_histogram(cluster, environ, topology, instance, role=None):\n  '''\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param instance:\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      instance=instance)\n\n  if role is not None:\n    params['role'] = role\n\n  request_url = tornado.httputil.url_concat(\n      create_url(HISTOGRAM_URL_FMT), params)\n\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef run_instance_jmap(cluster, environ, topology, instance, role=None):\n  '''\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param instance:\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      instance=instance)\n\n  if role is not None:\n    params['role'] = role\n\n  request_url = tornado.httputil.url_concat(\n      create_url(JMAP_URL_FMT), params)\n\n  if role is not None:\n    request_url = tornado.httputil.url_concat(request_url, dict(role=role))\n\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))", "response": "Runs a JMAP request for the specified instance."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the URL to download the container file.", "response": "def get_container_file_download_url(cluster, environ, topology, container,\n                                    path, role=None):\n  '''\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param container:\n  :param path:\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      container=container,\n      path=path)\n  if role is not None:\n    params['role'] = role\n\n  request_url = tornado.httputil.url_concat(\n      create_url(FILE_DOWNLOAD_URL_FMT), params)\n\n  if role is not None:\n    request_url = tornado.httputil.url_concat(request_url, dict(role=role))\n  return request_url"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget container file data from a container.", "response": "def get_container_file_data(cluster, environ, topology, container,\n                            path, offset, length, role=None):\n  '''\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param container:\n  :param path:\n  :param offset:\n  :param length:\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      container=container,\n      path=path,\n      offset=offset,\n      length=length)\n\n  if role is not None:\n    params['role'] = role\n\n  request_url = tornado.httputil.url_concat(\n      create_url(FILE_DATA_URL_FMT), params)\n\n  if role is not None:\n    request_url = tornado.httputil.url_concat(request_url, dict(role=role))\n\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_filestats(cluster, environ, topology, container, path, role=None):\n  '''\n  :param cluster:\n  :param environ:\n  :param topology:\n  :param container:\n  :param path:\n  :param role:\n  :return:\n  '''\n  params = dict(\n      cluster=cluster,\n      environ=environ,\n      topology=topology,\n      container=container,\n      path=path)\n\n  if role is not None:\n    params['role'] = role\n\n  request_url = tornado.httputil.url_concat(create_url(FILESTATS_URL_FMT), params)\n\n  raise tornado.gen.Return((yield fetch_url_as_json(request_url)))", "response": "Get all the files in a cluster."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget metrics for a given instance.", "response": "def fetch(self, cluster, metric, topology, component, instance, timerange, environ=None):\n    '''\n    :param cluster:\n    :param metric:\n    :param topology:\n    :param component:\n    :param instance:\n    :param timerange:\n    :param environ:\n    :return:\n    '''\n    components = [component] if component != \"*\" else (yield get_comps(cluster, environ, topology))\n\n    futures = []\n    for comp in components:\n      query = self.get_query(metric, comp, instance)\n      future = get_metrics(cluster, environ, topology, timerange, query)\n      futures.append(future)\n\n    results = yield futures\n\n    timelines = []\n    for result in results:\n      timelines.extend(result[\"timeline\"])\n\n    result = self.get_metric_response(timerange, timelines, False)\n\n    raise tornado.gen.Return(result)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the max metric for each component in the cluster.", "response": "def fetch_max(self, cluster, metric, topology, component, instance, timerange, environ=None):\n    '''\n    :param cluster:\n    :param metric:\n    :param topology:\n    :param component:\n    :param instance:\n    :param timerange:\n    :param environ:\n    :return:\n    '''\n    components = [component] if component != \"*\" else (yield get_comps(cluster, environ, topology))\n\n    result = {}\n    futures = []\n    for comp in components:\n      query = self.get_query(metric, comp, instance)\n      max_query = \"MAX(%s)\" % query\n      future = get_metrics(cluster, environ, topology, timerange, max_query)\n      futures.append(future)\n\n    results = yield futures\n\n    data = self.compute_max(results)\n\n    result = self.get_metric_response(timerange, data, True)\n\n    raise tornado.gen.Return(result)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef fetch_backpressure(self, cluster, metric, topology, component, instance, \\\n    timerange, is_max, environ=None):\n    '''\n    :param cluster:\n    :param metric:\n    :param topology:\n    :param component:\n    :param instance:\n    :param timerange:\n    :param isMax:\n    :param environ:\n    :return:\n    '''\n    instances = yield get_instances(cluster, environ, topology)\n    if component != \"*\":\n      filtered_inst = [instance for instance in instances if instance.split(\"_\")[2] == component]\n    else:\n      filtered_inst = instances\n\n    futures_dict = {}\n    for inst in filtered_inst:\n      query = queries.get(metric).format(inst)\n      futures_dict[inst] = get_metrics(cluster, environ, topology, timerange, query)\n\n    res = yield futures_dict\n\n    if not is_max:\n      timelines = []\n      for key in res:\n        result = res[key]\n        # Replacing stream manager instance name with component instance name\n        if len(result[\"timeline\"]) > 0:\n          result[\"timeline\"][0][\"instance\"] = key\n        timelines.extend(result[\"timeline\"])\n      result = self.get_metric_response(timerange, timelines, is_max)\n    else:\n      data = self.compute_max(res.values())\n      result = self.get_metric_response(timerange, data, is_max)\n\n    raise tornado.gen.Return(result)", "response": "Get backpressure metrics for a given component."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef compute_max(self, multi_ts):\n    '''\n    :param multi_ts:\n    :return:\n    '''\n    if len(multi_ts) > 0 and len(multi_ts[0][\"timeline\"]) > 0:\n      keys = multi_ts[0][\"timeline\"][0][\"data\"].keys()\n      timelines = ([res[\"timeline\"][0][\"data\"][key] for key in keys] for res in multi_ts)\n      values = (max(v) for v in zip(*timelines))\n      return dict(zip(keys, values))\n    return {}", "response": "Compute the maximum value of a multi - Timeseries"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a dictionary of the response from the metric service", "response": "def get_metric_response(self, timerange, data, isMax):\n    '''\n    :param timerange:\n    :param data:\n    :param isMax:\n    :return:\n    '''\n    if isMax:\n      return dict(\n          status=\"success\",\n          starttime=timerange[0],\n          endtime=timerange[1],\n          result=dict(timeline=[dict(data=data)])\n      )\n\n    return dict(\n        status=\"success\",\n        starttime=timerange[0],\n        endtime=timerange[1],\n        result=dict(timeline=data)\n    )"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_query(self, metric, component, instance):\n    '''\n    :param metric:\n    :param component:\n    :param instance:\n    :return:\n    '''\n    q = queries.get(metric)\n    return q.format(component, instance)", "response": "Get query string for a given metric component and instance."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nnormalizing raw result to table", "response": "def to_table(result):\n  ''' normalize raw result to table '''\n  max_count = 20\n  table, count = [], 0\n  for role, envs_topos in result.items():\n    for env, topos in envs_topos.items():\n      for topo in topos:\n        count += 1\n        if count > max_count:\n          continue\n        else:\n          table.append([role, env, topo])\n  header = ['role', 'env', 'topology']\n  rest_count = 0 if count <= max_count else count - max_count\n  return table, header, rest_count"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nshows topologies in a cluster", "response": "def show_cluster(cl_args, cluster):\n  ''' print topologies information to stdout '''\n  try:\n    result = tracker_access.get_cluster_topologies(cluster)\n    if not result:\n      Log.error('No topologies in cluster \\'%s\\'' % cluster)\n      return False\n    result = result[cluster]\n  except Exception:\n    Log.error(\"Fail to connect to tracker: \\'%s\\'\", cl_args[\"tracker_url\"])\n    return False\n  table, header, rest_count = to_table(result)\n  print('Topologies running in cluster \\'%s\\'' % cluster)\n  if rest_count:\n    print('  with %d more...' % rest_count)\n  print(tabulate(table, headers=header))\n  return True"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef show_cluster_role(cl_args, cluster, role):\n  ''' print topologies information to stdout '''\n  try:\n    result = tracker_access.get_cluster_role_topologies(cluster, role)\n    if not result:\n      Log.error('Unknown cluster/role \\'%s\\'' % '/'.join([cluster, role]))\n      return False\n    result = result[cluster]\n  except Exception:\n    Log.error(\"Fail to connect to tracker: \\'%s\\'\", cl_args[\"tracker_url\"])\n    return False\n  table, header, rest_count = to_table(result)\n  print('Topologies running in cluster \\'%s\\' submitted by \\'%s\\':' % (cluster, role))\n  if rest_count:\n    print('  with %d more...' % rest_count)\n  print(tabulate(table, headers=header))\n  return True", "response": "show topologies of a given cluster and role"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef show_cluster_role_env(cl_args, cluster, role, env):\n  ''' print topologies information to stdout '''\n  try:\n    result = tracker_access.get_cluster_role_env_topologies(cluster, role, env)\n    if not result:\n      Log.error('Unknown cluster/role/env \\'%s\\'' % '/'.join([cluster, role, env]))\n      return False\n    result = result[cluster]\n  except Exception:\n    Log.error(\"Fail to connect to tracker: \\'%s\\'\", cl_args[\"tracker_url\"])\n    return False\n  table, header, rest_count = to_table(result)\n  print('Topologies running in cluster \\'%s\\', submitted by \\'%s\\', and\\\n under environment \\'%s\\':' % (cluster, role, env))\n  if rest_count:\n    print('  with %d more...' % rest_count)\n  print(tabulate(table, headers=header))\n  return True", "response": "show topologies running in a cluster and under a specific environment"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nexecute a Heron class given the args and the jars needed for class path .", "response": "def heron_class(class_name, lib_jars, extra_jars=None, args=None, java_defines=None):\n  '''\n  Execute a heron class given the args and the jars needed for class path\n  :param class_name:\n  :param lib_jars:\n  :param extra_jars:\n  :param args:\n  :param java_defines:\n  :return:\n  '''\n  # default optional params to empty list if not provided\n  if extra_jars is None:\n    extra_jars = []\n  if args is None:\n    args = []\n  if java_defines is None:\n    java_defines = []\n\n  # Format all java -D options that need to be passed while running\n  # the class locally.\n  java_opts = ['-D' + opt for opt in java_defines]\n\n  # Construct the command line for the sub process to run\n  # Because of the way Python execute works,\n  # the java opts must be passed as part of the list\n  all_args = [config.get_java_path(), \"-client\", \"-Xmx1g\"] + \\\n             java_opts + \\\n             [\"-cp\", config.get_classpath(extra_jars + lib_jars)]\n\n  all_args += [class_name] + list(args)\n\n  # set heron_config environment variable\n  heron_env = os.environ.copy()\n  heron_env['HERON_OPTIONS'] = opts.get_heron_config()\n\n  # print the verbose message\n  Log.debug(\"Invoking class using command: ``%s''\", ' '.join(all_args))\n  Log.debug(\"Heron options: {%s}\", str(heron_env[\"HERON_OPTIONS\"]))\n\n  # invoke the command with subprocess and print error message, if any\n  process = subprocess.Popen(all_args, env=heron_env, stdout=subprocess.PIPE,\n                             stderr=subprocess.PIPE, bufsize=1)\n  # stdout message has the information Java program sends back\n  # stderr message has extra information, such as debugging message\n  return ProcessResult(process)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get(self, cluster, environ, topology, comp_name):\n    '''\n    :param cluster:\n    :param environ:\n    :param topology:\n    :param comp_name:\n    :return:\n    '''\n    start_time = time.time()\n    comp_names = []\n    if comp_name == \"All\":\n      lplan = yield access.get_logical_plan(cluster, environ, topology)\n      if not lplan:\n        self.write(dict())\n        return\n\n      if not 'spouts' in lplan or not 'bolts' in lplan:\n        self.write(dict())\n        return\n      comp_names = lplan['spouts'].keys()\n      comp_names.extend(lplan['bolts'].keys())\n    else:\n      comp_names = [comp_name]\n    exception_infos = dict()\n    for comp_name in comp_names:\n      exception_infos[comp_name] = yield access.get_component_exceptionsummary(\n          cluster, environ, topology, comp_name)\n\n    # Combine exceptions from multiple component\n    aggregate_exceptions = dict()\n    for comp_name, exception_logs in exception_infos.items():\n      for exception_log in exception_logs:\n        class_name = exception_log['class_name']\n        if class_name != '':\n          if not class_name in aggregate_exceptions:\n            aggregate_exceptions[class_name] = 0\n          aggregate_exceptions[class_name] += int(exception_log['count'])\n    # Put the exception value in a table\n    aggregate_exceptions_table = []\n    for key in aggregate_exceptions:\n      aggregate_exceptions_table.append([key, str(aggregate_exceptions[key])])\n    result = dict(\n        status=\"success\",\n        executiontime=time.time() - start_time,\n        result=aggregate_exceptions_table)\n    self.write(result)", "response": "Get the exception summary for a given component."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get(self):\n    '''\n    :return:\n    '''\n    # get all the topologies from heron nest\n    topologies = yield access.get_topologies_states()\n\n    result = dict()\n\n    # now convert some of the fields to be displayable\n    for cluster, cluster_value in topologies.items():\n      result[cluster] = dict()\n      for environ, environ_value in cluster_value.items():\n        result[cluster][environ] = dict()\n        for topology, topology_value in environ_value.items():\n          if \"jobname\" not in topology_value or topology_value[\"jobname\"] is None:\n            continue\n\n          if \"submission_time\" in topology_value:\n            topology_value[\"submission_time\"] = topology_value[\"submission_time\"]\n          else:\n            topology_value[\"submission_time\"] = '-'\n\n          result[cluster][environ][topology] = topology_value\n\n    self.write(result)", "response": "get all the topologies from heron nest\n   "}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets a new node from the cluster.", "response": "def get(self, cluster, environ, topology):\n    '''\n    :param cluster:\n    :param environ:\n    :param topology:\n    :return:\n    '''\n\n    start_time = time.time()\n    lplan = yield access.get_logical_plan(cluster, environ, topology)\n\n    # construct the result\n    result = dict(\n        status=\"success\",\n        message=\"\",\n        version=common.VERSION,\n        executiontime=time.time() - start_time,\n        result=lplan\n    )\n\n    self.write(result)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get(self, cluster, environ, topology):\n    '''\n    :param cluster:\n    :param environ:\n    :param topology:\n    :return:\n    '''\n\n    start_time = time.time()\n    pplan = yield access.get_physical_plan(cluster, environ, topology)\n\n    result_map = dict(\n        status=\"success\",\n        message=\"\",\n        version=common.VERSION,\n        executiontime=time.time() - start_time,\n        result=pplan\n    )\n\n    self.write(result_map)", "response": "Get a new cluster with physical plan."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets a single component from the cluster.", "response": "def get(self, cluster, environ, topology, component):\n    '''\n    :param cluster:\n    :param environ:\n    :param topology:\n    :param component:\n    :return:\n    '''\n    start_time = time.time()\n    futures = yield access.get_component_exceptions(cluster, environ, topology, component)\n    result_map = dict(\n        status='success',\n        executiontime=time.time() - start_time,\n        result=futures)\n    self.write(json.dumps(result_map))"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets the current cluster s physical plan and instance.", "response": "def get(self, cluster, environ, topology, instance):\n    '''\n    :param cluster:\n    :param environ:\n    :param topology:\n    :param instance:\n    :return:\n    '''\n    pplan = yield access.get_physical_plan(cluster, environ, topology)\n    host = pplan['stmgrs'][pplan['instances'][instance]['stmgrId']]['host']\n    result = json.loads((yield access.get_instance_pid(\n        cluster, environ, topology, instance)))\n    self.write('<pre><br/>$%s>: %s<br/><br/>%s</pre>' % (\n        host,\n        tornado.escape.xhtml_escape(result['command']),\n        tornado.escape.xhtml_escape(result['stdout'])))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef renderProcessStdErr(self, stderr_line):\n    retcode = self.process.poll()\n    if retcode is not None and status_type(retcode) == Status.InvocationError:\n      self._do_log(Log.error, stderr_line)\n    else:\n      self._do_print(sys.stderr, stderr_line)", "response": "render stderr of shelled - out process"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef renderProcessStdOut(self, stdout):\n    # since we render stdout line based on Java process return code,\n    # ``status'' has to be already set\n    assert self.status is not None\n    # remove pending newline\n    if self.status == Status.Ok:\n      self._do_log(Log.info, stdout)\n    elif self.status == Status.HeronError:\n      # remove last newline since logging will append newline\n      self._do_log(Log.error, stdout)\n    # No need to prefix [INFO] here. We want to display dry-run response in a clean way\n    elif self.status == Status.DryRun:\n      self._do_print(sys.stdout, stdout)\n    elif self.status == Status.InvocationError:\n      self._do_print(sys.stdout, stdout)\n    else:\n      raise RuntimeError(\n          \"Unknown status type of value %d. Expected value: %s\" % \\\n          (self.status.value, list(Status)))", "response": "render process standard output of process\n       "}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn true if the host is reachable.", "response": "def is_host_port_reachable(self):\n    \"\"\"\n    Returns true if the host is reachable. In some cases, it may not be reachable a tunnel\n    must be used.\n    \"\"\"\n    for hostport in self.hostportlist:\n      try:\n        socket.create_connection(hostport, StateManager.TIMEOUT_SECONDS)\n        return True\n      except:\n        LOG.info(\"StateManager %s Unable to connect to host: %s port %i\"\n                 % (self.name, hostport[0], hostport[1]))\n        continue\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\npicks an unused port.", "response": "def pick_unused_port(self):\n    \"\"\" Pick an unused port. There is a slight chance that this wont work. \"\"\"\n    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n    s.bind(('127.0.0.1', 0))\n    _, port = s.getsockname()\n    s.close()\n    return port"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nestablishing an ssh tunnel for each local host and port.", "response": "def establish_ssh_tunnel(self):\n    \"\"\"\n    Establish an ssh tunnel for each local host and port\n    that can be used to communicate with the state host.\n    \"\"\"\n    localportlist = []\n    for (host, port) in self.hostportlist:\n      localport = self.pick_unused_port()\n      self.tunnel.append(subprocess.Popen(\n          ('ssh', self.tunnelhost, '-NL127.0.0.1:%d:%s:%d' % (localport, host, port))))\n      localportlist.append(('127.0.0.1', localport))\n    return localportlist"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nremove the topology from the Zookeeper.", "response": "def delete_topology_from_zk(self, topologyName):\n    \"\"\"\n    Removes the topology entry from:\n    1. topologies list,\n    2. pplan,\n    3. execution_state, and\n    \"\"\"\n    self.delete_pplan(topologyName)\n    self.delete_execution_state(topologyName)\n    self.delete_topology(topologyName)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef monitor(self):\n\n    def trigger_watches_based_on_files(watchers, path, directory, ProtoClass):\n      \"\"\"\n      For all the topologies in the watchers, check if the data\n      in directory has changed. Trigger the callback if it has.\n      \"\"\"\n      for topology, callbacks in watchers.items():\n        file_path = os.path.join(path, topology)\n        data = \"\"\n        if os.path.exists(file_path):\n          with open(os.path.join(path, topology)) as f:\n            data = f.read()\n        if topology not in directory or data != directory[topology]:\n          proto_object = ProtoClass()\n          proto_object.ParseFromString(data)\n          for callback in callbacks:\n            callback(proto_object)\n          directory[topology] = data\n\n    while not self.monitoring_thread_stop_signal:\n      topologies_path = self.get_topologies_path()\n\n      topologies = []\n      if os.path.isdir(topologies_path):\n        topologies = list(filter(\n            lambda f: os.path.isfile(os.path.join(topologies_path, f)),\n            os.listdir(topologies_path)))\n      if set(topologies) != set(self.topologies_directory):\n        for callback in self.topologies_watchers:\n          callback(topologies)\n      self.topologies_directory = topologies\n\n      trigger_watches_based_on_files(\n          self.topology_watchers, topologies_path, self.topologies_directory, Topology)\n\n      # Get the directory name for execution state\n      execution_state_path = os.path.dirname(self.get_execution_state_path(\"\"))\n      trigger_watches_based_on_files(\n          self.execution_state_watchers, execution_state_path,\n          self.execution_state_directory, ExecutionState)\n\n      # Get the directory name for packing_plan\n      packing_plan_path = os.path.dirname(self.get_packing_plan_path(\"\"))\n      trigger_watches_based_on_files(\n          self.packing_plan_watchers, packing_plan_path, self.packing_plan_directory, PackingPlan)\n\n      # Get the directory name for pplan\n      pplan_path = os.path.dirname(self.get_pplan_path(\"\"))\n      trigger_watches_based_on_files(\n          self.pplan_watchers, pplan_path,\n          self.pplan_directory, PhysicalPlan)\n\n      # Get the directory name for tmaster\n      tmaster_path = os.path.dirname(self.get_tmaster_path(\"\"))\n      trigger_watches_based_on_files(\n          self.tmaster_watchers, tmaster_path,\n          self.tmaster_directory, TMasterLocation)\n\n      # Get the directory name for scheduler location\n      scheduler_location_path = os.path.dirname(self.get_scheduler_location_path(\"\"))\n      trigger_watches_based_on_files(\n          self.scheduler_location_watchers, scheduler_location_path,\n          self.scheduler_location_directory, SchedulerLocation)\n\n      # Sleep for some time\n      self.event.wait(timeout=5)", "response": "Monitor the rootpath and call the callback functions for each topology in the directory."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets topology from file", "response": "def get_topology(self, topologyName, callback=None):\n    \"\"\"get topology\"\"\"\n    if callback:\n      self.topology_watchers[topologyName].append(callback)\n    else:\n      topology_path = self.get_topology_path(topologyName)\n      with open(topology_path) as f:\n        data = f.read()\n        topology = Topology()\n        topology.ParseFromString(data)\n        return topology"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_pplan(self, topologyName, callback=None):\n    if callback:\n      self.pplan_watchers[topologyName].append(callback)\n    else:\n      pplan_path = self.get_pplan_path(topologyName)\n      with open(pplan_path) as f:\n        data = f.read()\n        pplan = PhysicalPlan()\n        pplan.ParseFromString(data)\n        return pplan", "response": "Get a physical plan of a topology"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting the execution state of a topology", "response": "def get_execution_state(self, topologyName, callback=None):\n    \"\"\"\n    Get execution state\n    \"\"\"\n    if callback:\n      self.execution_state_watchers[topologyName].append(callback)\n    else:\n      execution_state_path = self.get_execution_state_path(topologyName)\n      with open(execution_state_path) as f:\n        data = f.read()\n        executionState = ExecutionState()\n        executionState.ParseFromString(data)\n        return executionState"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting the TMasterLocation object for a given topology name.", "response": "def get_tmaster(self, topologyName, callback=None):\n    \"\"\"\n    Get tmaster\n    \"\"\"\n    if callback:\n      self.tmaster_watchers[topologyName].append(callback)\n    else:\n      tmaster_path = self.get_tmaster_path(topologyName)\n      with open(tmaster_path) as f:\n        data = f.read()\n        tmaster = TMasterLocation()\n        tmaster.ParseFromString(data)\n        return tmaster"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create_socket_options():\n  sys_config = system_config.get_sys_config()\n  opt_list = [const.INSTANCE_NETWORK_WRITE_BATCH_SIZE_BYTES,\n              const.INSTANCE_NETWORK_WRITE_BATCH_TIME_MS,\n              const.INSTANCE_NETWORK_READ_BATCH_SIZE_BYTES,\n              const.INSTANCE_NETWORK_READ_BATCH_TIME_MS,\n              const.INSTANCE_NETWORK_OPTIONS_SOCKET_RECEIVED_BUFFER_SIZE_BYTES,\n              const.INSTANCE_NETWORK_OPTIONS_SOCKET_SEND_BUFFER_SIZE_BYTES]\n\n  Log.debug(\"In create_socket_options()\")\n  try:\n    value_lst = [int(sys_config[opt]) for opt in opt_list]\n    sock_opt = SocketOptions(*value_lst)\n    return sock_opt\n  except ValueError as e:\n    # couldn't convert to int\n    raise ValueError(\"Invalid value in sys_config: %s\" % str(e))\n  except KeyError as e:\n    # option key was not found\n    raise KeyError(\"Incomplete sys_config: %s\" % str(e))", "response": "Creates SocketOptions object from a given sys_config dict"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ntake a class __dict__ and returns a HeronComponentSpec entries", "response": "def class_dict_to_specs(mcs, class_dict):\n    \"\"\"Takes a class `__dict__` and returns `HeronComponentSpec` entries\"\"\"\n    specs = {}\n\n    for name, spec in class_dict.items():\n      if isinstance(spec, HeronComponentSpec):\n        # Use the variable name as the specification name.\n        if spec.name is None:\n          spec.name = name\n        if spec.name in specs:\n          raise ValueError(\"Duplicate component name: %s\" % spec.name)\n        else:\n          specs[spec.name] = spec\n    return specs"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ntakes a class dict and returns a map containing topology - wide configuration.", "response": "def class_dict_to_topo_config(mcs, class_dict):\n    \"\"\"\n    Takes a class `__dict__` and returns a map containing topology-wide\n    configuration.\n\n    The returned dictionary is a sanitized `dict` of type `<str ->\n    (str|object)>`.\n\n    This classmethod firsts insert default topology configuration and then\n    overrides it with a given topology-wide configuration. Note that this\n    configuration will be overriden by a component-specific configuration at\n    runtime.\n    \"\"\"\n    topo_config = {}\n\n    # add defaults\n    topo_config.update(mcs.DEFAULT_TOPOLOGY_CONFIG)\n\n    for name, custom_config in class_dict.items():\n      if name == 'config' and isinstance(custom_config, dict):\n        sanitized_dict = mcs._sanitize_config(custom_config)\n        topo_config.update(sanitized_dict)\n\n    return topo_config"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ninitialize a topology protobuf", "response": "def init_topology(mcs, classname, class_dict):\n    \"\"\"Initializes a topology protobuf\"\"\"\n    if classname == 'Topology':\n      # Base class can't initialize protobuf\n      return\n    heron_options = TopologyType.get_heron_options_from_env()\n    initial_state = heron_options.get(\"cmdline.topology.initial.state\", \"RUNNING\")\n    tmp_directory = heron_options.get(\"cmdline.topologydefn.tmpdirectory\")\n    if tmp_directory is None:\n      raise RuntimeError(\"Topology definition temp directory not specified\")\n\n    topology_name = heron_options.get(\"cmdline.topology.name\", classname)\n    topology_id = topology_name + str(uuid.uuid4())\n\n    # create protobuf\n    topology = topology_pb2.Topology()\n    topology.id = topology_id\n    topology.name = topology_name\n    topology.state = topology_pb2.TopologyState.Value(initial_state)\n    topology.topology_config.CopyFrom(TopologyType.get_topology_config_protobuf(class_dict))\n\n    TopologyType.add_bolts_and_spouts(topology, class_dict)\n\n    class_dict['topology_name'] = topology_name\n    class_dict['topology_id'] = topology_id\n    class_dict['protobuf_topology'] = topology\n    class_dict['topologydefn_tmpdir'] = tmp_directory\n    class_dict['heron_runtime_options'] = heron_options"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_heron_options_from_env():\n    heron_options_raw = os.environ.get(\"HERON_OPTIONS\")\n    if heron_options_raw is None:\n      raise RuntimeError(\"HERON_OPTIONS environment variable not found\")\n\n    options = {}\n    for option_line in heron_options_raw.replace(\"%%%%\", \" \").split(','):\n      key, sep, value = option_line.partition(\"=\")\n      if sep:\n        options[key] = value\n      else:\n        raise ValueError(\"Invalid HERON_OPTIONS part %r\" % option_line)\n    return options", "response": "Retrieves heron options from the HERON_OPTIONS environment variable."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding a set of HeronComponentSpecs to the topology.", "response": "def add_spec(self, *specs):\n    \"\"\"Add specs to the topology\n\n    :type specs: HeronComponentSpec\n    :param specs: specs to add to the topology\n    \"\"\"\n    for spec in specs:\n      if not isinstance(spec, HeronComponentSpec):\n        raise TypeError(\"Argument to add_spec needs to be HeronComponentSpec, given: %s\"\n                        % str(spec))\n      if spec.name is None:\n        raise ValueError(\"TopologyBuilder cannot take a spec without name\")\n      if spec.name == \"config\":\n        raise ValueError(\"config is a reserved name\")\n      if spec.name in self._specs:\n        raise ValueError(\"Attempting to add duplicate spec name: %r %r\" % (spec.name, spec))\n\n      self._specs[spec.name] = spec"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds a spout to the topology", "response": "def add_spout(self, name, spout_cls, par, config=None, optional_outputs=None):\n    \"\"\"Add a spout to the topology\"\"\"\n    spout_spec = spout_cls.spec(name=name, par=par, config=config,\n                                optional_outputs=optional_outputs)\n    self.add_spec(spout_spec)\n    return spout_spec"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef add_bolt(self, name, bolt_cls, par, inputs, config=None, optional_outputs=None):\n    bolt_spec = bolt_cls.spec(name=name, par=par, inputs=inputs, config=config,\n                              optional_outputs=optional_outputs)\n    self.add_spec(bolt_spec)\n    return bolt_spec", "response": "Add a bolt to the topology"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_config(self, config):\n    if not isinstance(config, dict):\n      raise TypeError(\"Argument to set_config needs to be dict, given: %s\" % str(config))\n    self._topology_config = config", "response": "Set the topology - wide configuration to the topology - wide configuration."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nbuild the topology and submits to the destination", "response": "def build_and_submit(self):\n    \"\"\"Builds the topology and submits to the destination\"\"\"\n    class_dict = self._construct_topo_class_dict()\n    topo_cls = TopologyType(self.topology_name, (Topology,), class_dict)\n    topo_cls.write()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nfetch the given url and convert the response to json.", "response": "def fetch_url_as_json(fetch_url, default_value=None):\n  '''\n  Fetch the given url and convert the response to json.\n  :param fetch_url: URL to fetch\n  :param default_value: value to return in case of failure\n  :return:\n  '''\n  # assign empty dict for optional param\n  if default_value is None:\n    default_value = dict()\n\n  Log.debug(\"fetching url %s\", fetch_url)\n  ret = default_value\n\n  # time the duration of the fetch\n  start = time.time()\n\n  # fetch the URL asynchronously\n  http_response = yield tornado.httpclient.AsyncHTTPClient().fetch(fetch_url)\n\n  # handle http errors, and return if any\n  if http_response.error:\n    Log.error(\"Unable to get response from %s. Error %s\", fetch_url, http_response.error)\n    raise tornado.gen.Return(ret)\n\n  # load response and handle return errors, if any\n  response = json.loads(http_response.body)\n  if not 'result' in response:\n    Log.error(\"Empty response from %s\", fetch_url)\n    raise tornado.gen.Return(ret)\n\n  # get the response and execution time on server side\n  ret = response['result']\n  execution = 1000 * response['executiontime']\n\n  # calculate the time\n  end = time.time()\n  duration = 1000 * (end - start)\n\n  Log.debug(\"TIME: url fetch took %.2f ms server time %s\", execution, fetch_url)\n  Log.debug(\"TIME: url fetch took %.2f ms round trip  %s\", duration, fetch_url)\n\n  # convert future to value\n  raise tornado.gen.Return(ret)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nmaps from query parameter to query name", "response": "def queries_map():\n  \"\"\"map from query parameter to query name\"\"\"\n  qs = _all_metric_queries()\n  return dict(zip(qs[0], qs[1]) + zip(qs[2], qs[3]))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_clusters():\n  instance = tornado.ioloop.IOLoop.instance()\n  # pylint: disable=unnecessary-lambda\n  try:\n    return instance.run_sync(lambda: API.get_clusters())\n  except Exception:\n    Log.debug(traceback.format_exc())\n    raise", "response": "Synced API call to get all cluster names"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_topology_info(*args):\n  instance = tornado.ioloop.IOLoop.instance()\n  try:\n    return instance.run_sync(lambda: API.get_topology_info(*args))\n  except Exception:\n    Log.debug(traceback.format_exc())\n    raise", "response": "Synced API call to get topology information"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef configure(level=logging.INFO, logfile=None):\n\n  # Remove all the existing StreamHandlers to avoid duplicate\n  for handler in Log.handlers:\n    if isinstance(handler, logging.StreamHandler):\n      Log.handlers.remove(handler)\n\n  Log.setLevel(level)\n\n  # if logfile is specified, FileHandler is used\n  if logfile is not None:\n    log_format = \"[%(asctime)s] [%(levelname)s]: %(message)s\"\n    formatter = logging.Formatter(fmt=log_format, datefmt=date_format)\n    file_handler = logging.FileHandler(logfile)\n    file_handler.setFormatter(formatter)\n    Log.addHandler(file_handler)\n  # otherwise, use StreamHandler to output to stream (stdout, stderr...)\n  else:\n    log_format = \"[%(asctime)s] %(log_color)s[%(levelname)s]%(reset)s: %(message)s\"\n    # pylint: disable=redefined-variable-type\n    formatter = colorlog.ColoredFormatter(fmt=log_format, datefmt=date_format)\n    stream_handler = logging.StreamHandler()\n    stream_handler.setFormatter(formatter)\n    Log.addHandler(stream_handler)", "response": "Configure logger which dumps log on terminal"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef init_rotating_logger(level, logfile, max_files, max_bytes):\n  logging.basicConfig()\n\n  root_logger = logging.getLogger()\n  log_format = \"[%(asctime)s] [%(levelname)s] %(filename)s: %(message)s\"\n\n  root_logger.setLevel(level)\n  handler = RotatingFileHandler(logfile, maxBytes=max_bytes, backupCount=max_files)\n  handler.setFormatter(logging.Formatter(fmt=log_format, datefmt=date_format))\n  root_logger.addHandler(handler)\n\n  for handler in root_logger.handlers:\n    root_logger.debug(\"Associated handlers - \" + str(handler))\n    if isinstance(handler, logging.StreamHandler):\n      root_logger.debug(\"Removing StreamHandler: \" + str(handler))\n      root_logger.handlers.remove(handler)", "response": "Initializes a rotating logger with the given level logfile and max_bytes."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_logging_level(cl_args):\n  if 'verbose' in cl_args and cl_args['verbose']:\n    configure(logging.DEBUG)\n  else:\n    configure(logging.INFO)", "response": "simply set verbose level based on command - line args\n "}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _get_spout(self):\n    spout = topology_pb2.Spout()\n    spout.comp.CopyFrom(self._get_base_component())\n\n    # Add output streams\n    self._add_out_streams(spout)\n    return spout", "response": "Returns a Spout protobuf message"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _get_bolt(self):\n    bolt = topology_pb2.Bolt()\n    bolt.comp.CopyFrom(self._get_base_component())\n\n    # Add streams\n    self._add_in_streams(bolt)\n    self._add_out_streams(bolt)\n    return bolt", "response": "Returns Bolt protobuf message"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning Component protobuf message", "response": "def _get_base_component(self):\n    \"\"\"Returns Component protobuf message\"\"\"\n    comp = topology_pb2.Component()\n    comp.name = self.name\n    comp.spec = topology_pb2.ComponentObjectSpec.Value(\"PYTHON_CLASS_NAME\")\n    comp.class_name = self.python_class_path\n    comp.config.CopyFrom(self._get_comp_config())\n    return comp"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the component - specific Config protobuf message that is used by the component.", "response": "def _get_comp_config(self):\n    \"\"\"Returns component-specific Config protobuf message\n\n    It first adds ``topology.component.parallelism``, and is overriden by\n    a user-defined component-specific configuration, specified by spec().\n    \"\"\"\n    proto_config = topology_pb2.Config()\n\n    # first add parallelism\n    key = proto_config.kvs.add()\n    key.key = TOPOLOGY_COMPONENT_PARALLELISM\n    key.value = str(self.parallelism)\n    key.type = topology_pb2.ConfigValueType.Value(\"STRING_VALUE\")\n\n    # iterate through self.custom_config\n    if self.custom_config is not None:\n      sanitized = self._sanitize_config(self.custom_config)\n      for key, value in sanitized.items():\n        if isinstance(value, str):\n          kvs = proto_config.kvs.add()\n          kvs.key = key\n          kvs.value = value\n          kvs.type = topology_pb2.ConfigValueType.Value(\"STRING_VALUE\")\n        else:\n          # need to serialize\n          kvs = proto_config.kvs.add()\n          kvs.key = key\n          kvs.serialized_value = default_serializer.serialize(value)\n          kvs.type = topology_pb2.ConfigValueType.Value(\"PYTHON_SERIALIZED_VALUE\")\n\n    return proto_config"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _sanitize_config(custom_config):\n    if not isinstance(custom_config, dict):\n      raise TypeError(\"Component-specific configuration must be given as a dict type, given: %s\"\n                      % str(type(custom_config)))\n    sanitized = {}\n    for key, value in custom_config.items():\n      if not isinstance(key, str):\n        raise TypeError(\"Key for component-specific configuration must be string, given: %s:%s\"\n                        % (str(type(key)), str(key)))\n\n      if isinstance(value, bool):\n        sanitized[key] = \"true\" if value else \"false\"\n      elif isinstance(value, (str, int, float)):\n        sanitized[key] = str(value)\n      else:\n        sanitized[key] = value\n\n    return sanitized", "response": "Checks whether a given dictionary is sane and returns a sanitized dict that is suitable for use in a protobuf message."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding inputs to a given protobuf Bolt message", "response": "def _add_in_streams(self, bolt):\n    \"\"\"Adds inputs to a given protobuf Bolt message\"\"\"\n    if self.inputs is None:\n      return\n    # sanitize inputs and get a map <GlobalStreamId -> Grouping>\n    input_dict = self._sanitize_inputs()\n\n    for global_streamid, gtype in input_dict.items():\n      in_stream = bolt.inputs.add()\n      in_stream.stream.CopyFrom(self._get_stream_id(global_streamid.component_id,\n                                                    global_streamid.stream_id))\n      if isinstance(gtype, Grouping.FIELDS):\n        # it's a field grouping\n        in_stream.gtype = gtype.gtype\n        in_stream.grouping_fields.CopyFrom(self._get_stream_schema(gtype.fields))\n      elif isinstance(gtype, Grouping.CUSTOM):\n        # it's a custom grouping\n        in_stream.gtype = gtype.gtype\n        in_stream.custom_grouping_object = gtype.python_serialized\n        in_stream.type = topology_pb2.CustomGroupingObjectType.Value(\"PYTHON_OBJECT\")\n      else:\n        in_stream.gtype = gtype"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _sanitize_inputs(self):\n    ret = {}\n    if self.inputs is None:\n      return\n\n    if isinstance(self.inputs, dict):\n      # inputs are dictionary, must be either <HeronComponentSpec -> Grouping> or\n      # <GlobalStreamId -> Grouping>\n      for key, grouping in self.inputs.items():\n        if not Grouping.is_grouping_sane(grouping):\n          raise ValueError('A given grouping is not supported')\n        if isinstance(key, HeronComponentSpec):\n          # use default streamid\n          if key.name is None:\n            # should not happen as TopologyType metaclass sets name attribute\n            # before calling this method\n            raise RuntimeError(\"In _sanitize_inputs(): HeronComponentSpec doesn't have a name\")\n          global_streamid = GlobalStreamId(key.name, Stream.DEFAULT_STREAM_ID)\n          ret[global_streamid] = grouping\n        elif isinstance(key, GlobalStreamId):\n          ret[key] = grouping\n        else:\n          raise ValueError(\"%s is not supported as a key to inputs\" % str(key))\n    elif isinstance(self.inputs, (list, tuple)):\n      # inputs are lists, must be either a list of HeronComponentSpec or GlobalStreamId\n      # will use SHUFFLE grouping\n      for input_obj in self.inputs:\n        if isinstance(input_obj, HeronComponentSpec):\n          if input_obj.name is None:\n            # should not happen as TopologyType metaclass sets name attribute\n            # before calling this method\n            raise RuntimeError(\"In _sanitize_inputs(): HeronComponentSpec doesn't have a name\")\n          global_streamid = GlobalStreamId(input_obj.name, Stream.DEFAULT_STREAM_ID)\n          ret[global_streamid] = Grouping.SHUFFLE\n        elif isinstance(input_obj, GlobalStreamId):\n          ret[input_obj] = Grouping.SHUFFLE\n        else:\n          raise ValueError(\"%s is not supported as an input\" % str(input_obj))\n    else:\n      raise TypeError(\"Inputs must be a list, dict, or None, given: %s\" % str(self.inputs))\n\n    return ret", "response": "Sanitize the inputs of the HeronComponentSpec and return a map of GlobalStreamId - > Grouping"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _add_out_streams(self, spbl):\n    if self.outputs is None:\n      return\n\n    # sanitize outputs and get a map <stream_id -> out fields>\n    output_map = self._sanitize_outputs()\n\n    for stream_id, out_fields in output_map.items():\n      out_stream = spbl.outputs.add()\n      out_stream.stream.CopyFrom(self._get_stream_id(self.name, stream_id))\n      out_stream.schema.CopyFrom(self._get_stream_schema(out_fields))", "response": "Adds outputs to a given protobuf Bolt or Spout message"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _sanitize_outputs(self):\n    ret = {}\n    if self.outputs is None:\n      return\n\n    if not isinstance(self.outputs, (list, tuple)):\n      raise TypeError(\"Argument to outputs must be either list or tuple, given: %s\"\n                      % str(type(self.outputs)))\n\n    for output in self.outputs:\n      if not isinstance(output, (str, Stream)):\n        raise TypeError(\"Outputs must be a list of strings or Streams, given: %s\" % str(output))\n\n      if isinstance(output, str):\n        # it's a default stream\n        if Stream.DEFAULT_STREAM_ID not in ret:\n          ret[Stream.DEFAULT_STREAM_ID] = list()\n        ret[Stream.DEFAULT_STREAM_ID].append(output)\n      else:\n        # output is a Stream object\n        if output.stream_id == Stream.DEFAULT_STREAM_ID and Stream.DEFAULT_STREAM_ID in ret:\n          # some default stream fields are already in there\n          ret[Stream.DEFAULT_STREAM_ID].extend(output.fields)\n        else:\n          ret[output.stream_id] = output.fields\n    return ret", "response": "Sanitize the outputs and returns a map <stream_id > = list of output fields"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a set of output stream ids registered for this component", "response": "def get_out_streamids(self):\n    \"\"\"Returns a set of output stream ids registered for this component\"\"\"\n    if self.outputs is None:\n      return set()\n\n    if not isinstance(self.outputs, (list, tuple)):\n      raise TypeError(\"Argument to outputs must be either list or tuple, given: %s\"\n                      % str(type(self.outputs)))\n    ret_lst = []\n    for output in self.outputs:\n      if not isinstance(output, (str, Stream)):\n        raise TypeError(\"Outputs must be a list of strings or Streams, given: %s\" % str(output))\n      ret_lst.append(Stream.DEFAULT_STREAM_ID if isinstance(output, str) else output.stream_id)\n    return set(ret_lst)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _get_stream_id(comp_name, stream_id):\n    proto_stream_id = topology_pb2.StreamId()\n    proto_stream_id.id = stream_id\n    proto_stream_id.component_name = comp_name\n    return proto_stream_id", "response": "Returns a StreamId protobuf message"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a StreamSchema protobuf message for the given fields.", "response": "def _get_stream_schema(fields):\n    \"\"\"Returns a StreamSchema protobuf message\"\"\"\n    stream_schema = topology_pb2.StreamSchema()\n    for field in fields:\n      key = stream_schema.keys.add()\n      key.key = field\n      key.type = topology_pb2.Type.Value(\"OBJECT\")\n\n    return stream_schema"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef component_id(self):\n    if isinstance(self._component_id, HeronComponentSpec):\n      if self._component_id.name is None:\n        # HeronComponentSpec instance's name attribute might not be available until\n        # TopologyType metaclass finally sets it. This statement is to support __eq__(),\n        # __hash__() and __str__() methods with safety, as raising Exception is not\n        # appropriate this case.\n        return \"<No name available for HeronComponentSpec yet, uuid: %s>\" % self._component_id.uuid\n      return self._component_id.name\n    elif isinstance(self._component_id, str):\n      return self._component_id\n    else:\n      raise ValueError(\"Component Id for this GlobalStreamId is not properly set: <%s:%s>\"\n                       % (str(type(self._component_id)), str(self._component_id)))", "response": "Returns the component_id of this GlobalStreamId"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting an error message to the output buffer.", "response": "def write_error(self, status_code, **kwargs):\n    '''\n    :param status_code:\n    :param kwargs:\n    :return:\n    '''\n    if \"exc_info\" in kwargs:\n      exc_info = kwargs[\"exc_info\"]\n      error = exc_info[1]\n\n      errormessage = \"%s: %s\" % (status_code, error)\n      self.render(\"error.html\", errormessage=errormessage)\n    else:\n      errormessage = \"%s\" % (status_code)\n      self.render(\"error.html\", errormessage=errormessage)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef register_metric(self, name, metric, time_bucket_in_sec):\n    collector = self.get_metrics_collector()\n    collector.register_metric(name, metric, time_bucket_in_sec)", "response": "Registers a new metric to this context"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_sources(self, component_id):\n    # this is necessary because protobuf message is not hashable\n    StreamId = namedtuple('StreamId', 'id, component_name')\n    if component_id in self.inputs:\n      ret = {}\n      for istream in self.inputs.get(component_id):\n        key = StreamId(id=istream.stream.id, component_name=istream.stream.component_name)\n        ret[key] = istream.gtype\n      return ret\n    else:\n      return None", "response": "Returns the declared inputs to specified component_id"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_component_tasks(self, component_id):\n    ret = []\n    for task_id, comp_id in self.task_to_component_map.items():\n      if comp_id == component_id:\n        ret.append(task_id)\n    return ret", "response": "Returns the list of task ids allocated for the given component id"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add_task_hook(self, task_hook):\n    if not isinstance(task_hook, ITaskHook):\n      raise TypeError(\"In add_task_hook(): attempt to add non ITaskHook instance, given: %s\"\n                      % str(type(task_hook)))\n    self.task_hooks.append(task_hook)", "response": "Registers a specified task hook to this context\n   "}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning this context s metrics collector", "response": "def get_metrics_collector(self):\n    \"\"\"Returns this context's metrics collector\"\"\"\n    if self.metrics_collector is None or not isinstance(self.metrics_collector, MetricsCollector):\n      raise RuntimeError(\"Metrics collector is not registered in this context\")\n    return self.metrics_collector"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef invoke_hook_prepare(self):\n    for task_hook in self.task_hooks:\n      task_hook.prepare(self.get_cluster_config(), self)", "response": "invoke task hooks for after the spout s initialize method"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef invoke_hook_emit(self, values, stream_id, out_tasks):\n    if len(self.task_hooks) > 0:\n      emit_info = EmitInfo(values=values, stream_id=stream_id,\n                           task_id=self.get_task_id(), out_tasks=out_tasks)\n      for task_hook in self.task_hooks:\n        task_hook.emit(emit_info)", "response": "invoke task hooks for every time a tuple is emitted in spout or bolt"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef invoke_hook_spout_ack(self, message_id, complete_latency_ns):\n    if len(self.task_hooks) > 0:\n      spout_ack_info = SpoutAckInfo(message_id=message_id,\n                                    spout_task_id=self.get_task_id(),\n                                    complete_latency_ms=complete_latency_ns *\n                                    system_constants.NS_TO_MS)\n      for task_hook in self.task_hooks:\n        task_hook.spout_ack(spout_ack_info)", "response": "invoke task hooks for every time spout acks a tuple"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef invoke_hook_spout_fail(self, message_id, fail_latency_ns):\n    if len(self.task_hooks) > 0:\n      spout_fail_info = SpoutFailInfo(message_id=message_id,\n                                      spout_task_id=self.get_task_id(),\n                                      fail_latency_ms=fail_latency_ns * system_constants.NS_TO_MS)\n      for task_hook in self.task_hooks:\n        task_hook.spout_fail(spout_fail_info)", "response": "invoke task hooks for every time spout fails a tuple\n   "}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef invoke_hook_bolt_execute(self, heron_tuple, execute_latency_ns):\n    if len(self.task_hooks) > 0:\n      bolt_execute_info = \\\n        BoltExecuteInfo(heron_tuple=heron_tuple,\n                        executing_task_id=self.get_task_id(),\n                        execute_latency_ms=execute_latency_ns * system_constants.NS_TO_MS)\n      for task_hook in self.task_hooks:\n        task_hook.bolt_execute(bolt_execute_info)", "response": "invoke task hooks for every time bolt processes a tuple"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ninvoking task hooks for every time bolt acks a tuple", "response": "def invoke_hook_bolt_ack(self, heron_tuple, process_latency_ns):\n    \"\"\"invoke task hooks for every time bolt acks a tuple\n\n    :type heron_tuple: HeronTuple\n    :param heron_tuple: tuple that is acked\n    :type process_latency_ns: float\n    :param process_latency_ns: process latency in nano seconds\n    \"\"\"\n    if len(self.task_hooks) > 0:\n      bolt_ack_info = BoltAckInfo(heron_tuple=heron_tuple,\n                                  acking_task_id=self.get_task_id(),\n                                  process_latency_ms=process_latency_ns * system_constants.NS_TO_MS)\n      for task_hook in self.task_hooks:\n        task_hook.bolt_ack(bolt_ack_info)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ninvoking task hooks for every time bolt fails a tuple", "response": "def invoke_hook_bolt_fail(self, heron_tuple, fail_latency_ns):\n    \"\"\"invoke task hooks for every time bolt fails a tuple\n\n    :type heron_tuple: HeronTuple\n    :param heron_tuple: tuple that is failed\n    :type fail_latency_ns: float\n    :param fail_latency_ns: fail latency in nano seconds\n    \"\"\"\n    if len(self.task_hooks) > 0:\n      bolt_fail_info = BoltFailInfo(heron_tuple=heron_tuple,\n                                    failing_task_id=self.get_task_id(),\n                                    fail_latency_ms=fail_latency_ns * system_constants.NS_TO_MS)\n      for task_hook in self.task_hooks:\n        task_hook.bolt_fail(bolt_fail_info)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a subparser for the submit command", "response": "def create_parser(subparsers):\n  '''\n  Create a subparser for the submit command\n  :param subparsers:\n  :return:\n  '''\n  parser = subparsers.add_parser(\n      'submit',\n      help='Submit a topology',\n      usage=\"%(prog)s [options] cluster/[role]/[env] \" + \\\n            \"topology-file-name topology-class-name [topology-args]\",\n      add_help=True\n  )\n\n  cli_args.add_titles(parser)\n  cli_args.add_cluster_role_env(parser)\n  cli_args.add_topology_file(parser)\n  cli_args.add_topology_class(parser)\n  cli_args.add_config(parser)\n  cli_args.add_deactive_deploy(parser)\n  cli_args.add_dry_run(parser)\n  cli_args.add_extra_launch_classpath(parser)\n  cli_args.add_release_yaml_file(parser)\n  cli_args.add_service_url(parser)\n  cli_args.add_system_property(parser)\n  cli_args.add_verbose(parser)\n\n  parser.set_defaults(subcommand='submit')\n  return parser"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nlaunch a topology given a topology jar its definition file and configurations This function is used to launch a topology given a topology jar its definition file and configurations This function is used to launch a topology given a topology jar and its configuration file and the configuration file and the configuration file.", "response": "def launch_a_topology(cl_args, tmp_dir, topology_file, topology_defn_file, topology_name):\n  '''\n  Launch a topology given topology jar, its definition file and configurations\n  :param cl_args:\n  :param tmp_dir:\n  :param topology_file:\n  :param topology_defn_file:\n  :param topology_name:\n  :return:\n  '''\n  # get the normalized path for topology.tar.gz\n  topology_pkg_path = config.normalized_class_path(os.path.join(tmp_dir, 'topology.tar.gz'))\n\n  # get the release yaml file\n  release_yaml_file = cl_args['release_yaml_file']\n\n  # create a tar package with the cluster configuration and generated config files\n  config_path = cl_args['config_path']\n  tar_pkg_files = [topology_file, topology_defn_file]\n  generated_config_files = [release_yaml_file, cl_args['override_config_file']]\n\n  config.create_tar(topology_pkg_path, tar_pkg_files, config_path, generated_config_files)\n\n  # pass the args to submitter main\n  args = [\n      \"--cluster\", cl_args['cluster'],\n      \"--role\", cl_args['role'],\n      \"--environment\", cl_args['environ'],\n      \"--submit_user\", cl_args['submit_user'],\n      \"--heron_home\", config.get_heron_dir(),\n      \"--config_path\", config_path,\n      \"--override_config_file\", cl_args['override_config_file'],\n      \"--release_file\", release_yaml_file,\n      \"--topology_package\", topology_pkg_path,\n      \"--topology_defn\", topology_defn_file,\n      \"--topology_bin\", os.path.basename(topology_file)   # pex/cpp file if pex/cpp specified\n  ]\n\n  if Log.getEffectiveLevel() == logging.DEBUG:\n    args.append(\"--verbose\")\n\n  if cl_args[\"dry_run\"]:\n    args.append(\"--dry_run\")\n    if \"dry_run_format\" in cl_args:\n      args += [\"--dry_run_format\", cl_args[\"dry_run_format\"]]\n\n  lib_jars = config.get_heron_libs(\n      jars.scheduler_jars() + jars.uploader_jars() + jars.statemgr_jars() + jars.packing_jars()\n  )\n  extra_jars = cl_args['extra_launch_classpath'].split(':')\n\n  # invoke the submitter to submit and launch the topology\n  main_class = 'org.apache.heron.scheduler.SubmitterMain'\n  res = execute.heron_class(\n      class_name=main_class,\n      lib_jars=lib_jars,\n      extra_jars=extra_jars,\n      args=args,\n      java_defines=[])\n\n  err_ctxt = \"Failed to launch topology '%s' %s\" % (topology_name, launch_mode_msg(cl_args))\n  succ_ctxt = \"Successfully launched topology '%s' %s\" % (topology_name, launch_mode_msg(cl_args))\n\n  res.add_context(err_ctxt, succ_ctxt)\n  return res"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef launch_topology_server(cl_args, topology_file, topology_defn_file, topology_name):\n  '''\n  Launch a topology given topology jar, its definition file and configurations\n  :param cl_args:\n  :param topology_file:\n  :param topology_defn_file:\n  :param topology_name:\n  :return:\n  '''\n  service_apiurl = cl_args['service_url'] + rest.ROUTE_SIGNATURES['submit'][1]\n  service_method = rest.ROUTE_SIGNATURES['submit'][0]\n  data = dict(\n      name=topology_name,\n      cluster=cl_args['cluster'],\n      role=cl_args['role'],\n      environment=cl_args['environ'],\n      user=cl_args['submit_user'],\n  )\n\n  Log.info(\"\" + str(cl_args))\n  overrides = dict()\n  if 'config_property' in cl_args:\n    overrides = config.parse_override_config(cl_args['config_property'])\n\n  if overrides:\n    data.update(overrides)\n\n  if cl_args['dry_run']:\n    data[\"dry_run\"] = True\n\n  files = dict(\n      definition=open(topology_defn_file, 'rb'),\n      topology=open(topology_file, 'rb'),\n  )\n\n  err_ctxt = \"Failed to launch topology '%s' %s\" % (topology_name, launch_mode_msg(cl_args))\n  succ_ctxt = \"Successfully launched topology '%s' %s\" % (topology_name, launch_mode_msg(cl_args))\n\n  try:\n    r = service_method(service_apiurl, data=data, files=files)\n    ok = r.status_code is requests.codes.ok\n    created = r.status_code is requests.codes.created\n    s = Status.Ok if created or ok else Status.HeronError\n    if s is Status.HeronError:\n      Log.error(r.json().get('message', \"Unknown error from API server %d\" % r.status_code))\n    elif ok:\n      # this case happens when we request a dry_run\n      print(r.json().get(\"response\"))\n  except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:\n    Log.error(err)\n    return SimpleResult(Status.HeronError, err_ctxt, succ_ctxt)\n  return SimpleResult(s, err_ctxt, succ_ctxt)", "response": "Launch a topology given a topology jar its definition file and configurations\n  Launch a topology given a topology jar its definition file and configurations\n "}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef launch_topologies(cl_args, topology_file, tmp_dir):\n  '''\n  Launch topologies\n  :param cl_args:\n  :param topology_file:\n  :param tmp_dir:\n  :return: list(Responses)\n  '''\n  # the submitter would have written the .defn file to the tmp_dir\n  defn_files = glob.glob(tmp_dir + '/*.defn')\n\n  if len(defn_files) == 0:\n    return SimpleResult(Status.HeronError, \"No topologies found under %s\" % tmp_dir)\n\n  results = []\n  for defn_file in defn_files:\n    # load the topology definition from the file\n    topology_defn = topology_pb2.Topology()\n    try:\n      handle = open(defn_file, \"rb\")\n      topology_defn.ParseFromString(handle.read())\n      handle.close()\n    except Exception as e:\n      err_context = \"Cannot load topology definition '%s': %s\" % (defn_file, e)\n      return SimpleResult(Status.HeronError, err_context)\n\n    # launch the topology\n    Log.info(\"Launching topology: \\'%s\\'%s\", topology_defn.name, launch_mode_msg(cl_args))\n\n    # check if we have to do server or direct based deployment\n    if cl_args['deploy_mode'] == config.SERVER_MODE:\n      res = launch_topology_server(\n          cl_args, topology_file, defn_file, topology_defn.name)\n    else:\n      res = launch_a_topology(\n          cl_args, tmp_dir, topology_file, defn_file, topology_defn.name)\n    results.append(res)\n\n  return results", "response": "Launch all topologies in the topology_file under the tmp_dir."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsubmitting a topology jar file to the Zookeeper.", "response": "def submit_fatjar(cl_args, unknown_args, tmp_dir):\n  '''\n  We use the packer to make a package for the jar and dump it\n  to a well-known location. We then run the main method of class\n  with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.\n\n  This will run the jar file with the topology_class_name. The submitter\n  inside will write out the topology defn file to a location that\n  we specify. Then we write the topology defn file to a well known\n  location. We then write to appropriate places in zookeeper\n  and launch the scheduler jobs\n  :param cl_args:\n  :param unknown_args:\n  :param tmp_dir:\n  :return:\n  '''\n  # execute main of the topology to create the topology definition\n  topology_file = cl_args['topology-file-name']\n\n  main_class = cl_args['topology-class-name']\n\n  res = execute.heron_class(\n      class_name=main_class,\n      lib_jars=config.get_heron_libs(jars.topology_jars()),\n      extra_jars=[topology_file],\n      args=tuple(unknown_args),\n      java_defines=cl_args['topology_main_jvm_property'])\n\n  result.render(res)\n\n  if not result.is_successful(res):\n    err_context = (\"Failed to create topology definition \" \\\n      \"file when executing class '%s' of file '%s'\") % (main_class, topology_file)\n    res.add_context(err_context)\n    return res\n\n  results = launch_topologies(cl_args, topology_file, tmp_dir)\n\n  return results"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef submit_tar(cl_args, unknown_args, tmp_dir):\n  '''\n  Extract and execute the java files inside the tar and then add topology\n  definition file created by running submitTopology\n\n  We use the packer to make a package for the tar and dump it\n  to a well-known location. We then run the main method of class\n  with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.\n  This will run the jar file with the topology class name.\n\n  The submitter inside will write out the topology defn file to a location\n  that we specify. Then we write the topology defn file to a well known\n  packer location. We then write to appropriate places in zookeeper\n  and launch the aurora jobs\n  :param cl_args:\n  :param unknown_args:\n  :param tmp_dir:\n  :return:\n  '''\n  # execute main of the topology to create the topology definition\n  topology_file = cl_args['topology-file-name']\n  java_defines = cl_args['topology_main_jvm_property']\n  main_class = cl_args['topology-class-name']\n  res = execute.heron_tar(\n      main_class,\n      topology_file,\n      tuple(unknown_args),\n      tmp_dir,\n      java_defines)\n\n  result.render(res)\n\n  if not result.is_successful(res):\n    err_context = (\"Failed to create topology definition \" \\\n      \"file when executing class '%s' of file '%s'\") % (main_class, topology_file)\n    res.add_context(err_context)\n    return res\n\n  return launch_topologies(cl_args, topology_file, tmp_dir)", "response": "Submit the topology file to the Zookeeper"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsubmit the topology to the scheduler", "response": "def run(command, parser, cl_args, unknown_args):\n  '''\n  Submits the topology to the scheduler\n    * Depending on the topology file name extension, we treat the file as a\n      fatjar (if the ext is .jar) or a tar file (if the ext is .tar/.tar.gz).\n    * We upload the topology file to the packer, update zookeeper and launch\n      scheduler jobs representing that topology\n    * You can see your topology in Heron UI\n  :param command:\n  :param parser:\n  :param cl_args:\n  :param unknown_args:\n  :return:\n  '''\n  Log.debug(\"Submit Args %s\", cl_args)\n\n  # get the topology file name\n  topology_file = cl_args['topology-file-name']\n\n  if urlparse.urlparse(topology_file).scheme:\n    cl_args['topology-file-name'] = download(topology_file, cl_args['cluster'])\n    topology_file = cl_args['topology-file-name']\n    Log.debug(\"download uri to local file: %s\", topology_file)\n\n  # check to see if the topology file exists\n  if not os.path.isfile(topology_file):\n    err_context = \"Topology file '%s' does not exist\" % topology_file\n    return SimpleResult(Status.InvocationError, err_context)\n\n  # check if it is a valid file type\n  jar_type = topology_file.endswith(\".jar\")\n  tar_type = topology_file.endswith(\".tar\") or topology_file.endswith(\".tar.gz\")\n  pex_type = topology_file.endswith(\".pex\")\n  cpp_type = topology_file.endswith(\".dylib\") or topology_file.endswith(\".so\")\n  if not (jar_type or tar_type or pex_type or cpp_type):\n    _, ext_name = os.path.splitext(topology_file)\n    err_context = \"Unknown file type '%s'. Please use .tar \"\\\n                  \"or .tar.gz or .jar or .pex or .dylib or .so file\"\\\n                  % ext_name\n    return SimpleResult(Status.InvocationError, err_context)\n\n  # check if extra launch classpath is provided and if it is validate\n  if cl_args['extra_launch_classpath']:\n    valid_classpath = classpath.valid_java_classpath(cl_args['extra_launch_classpath'])\n    if not valid_classpath:\n      err_context = \"One of jar or directory in extra launch classpath does not exist: %s\" % \\\n        cl_args['extra_launch_classpath']\n      return SimpleResult(Status.InvocationError, err_context)\n\n  # create a temporary directory for topology definition file\n  tmp_dir = tempfile.mkdtemp()\n  opts.cleaned_up_files.append(tmp_dir)\n\n  # if topology needs to be launched in deactivated state, do it so\n  if cl_args['deploy_deactivated']:\n    initial_state = topology_pb2.TopologyState.Name(topology_pb2.PAUSED)\n  else:\n    initial_state = topology_pb2.TopologyState.Name(topology_pb2.RUNNING)\n\n  # set the tmp dir and deactivated state in global options\n  opts.set_config('cmdline.topologydefn.tmpdirectory', tmp_dir)\n  opts.set_config('cmdline.topology.initial.state', initial_state)\n  opts.set_config('cmdline.topology.role', cl_args['role'])\n  opts.set_config('cmdline.topology.environment', cl_args['environ'])\n\n  # Use CLI release yaml file if the release_yaml_file config is empty\n  if not cl_args['release_yaml_file']:\n    cl_args['release_yaml_file'] = config.get_heron_release_file()\n\n  # check the extension of the file name to see if it is tar/jar file.\n  if jar_type:\n    return submit_fatjar(cl_args, unknown_args, tmp_dir)\n  elif tar_type:\n    return submit_tar(cl_args, unknown_args, tmp_dir)\n  elif cpp_type:\n    return submit_cpp(cl_args, unknown_args, tmp_dir)\n  else:\n    return submit_pex(cl_args, unknown_args, tmp_dir)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting method returns file data", "response": "def get(self):\n    \"\"\" get method \"\"\"\n    try:\n      cluster = self.get_argument_cluster()\n      role = self.get_argument_role()\n      environ = self.get_argument_environ()\n      topology_name = self.get_argument_topology()\n      container = self.get_argument(constants.PARAM_CONTAINER)\n      path = self.get_argument(constants.PARAM_PATH)\n      offset = self.get_argument_offset()\n      length = self.get_argument_length()\n      topology_info = self.tracker.getTopologyInfo(topology_name, cluster, role, environ)\n\n      stmgr_id = \"stmgr-\" + container\n      stmgr = topology_info[\"physical_plan\"][\"stmgrs\"][stmgr_id]\n      host = stmgr[\"host\"]\n      shell_port = stmgr[\"shell_port\"]\n      file_data_url = \"http://%s:%d/filedata/%s?offset=%s&length=%s\" % \\\n        (host, shell_port, path, offset, length)\n\n      http_client = tornado.httpclient.AsyncHTTPClient()\n      response = yield http_client.fetch(file_data_url)\n      self.write_success_response(json.loads(response.body))\n      self.finish()\n    except Exception as e:\n      Log.debug(traceback.format_exc())\n      self.write_error_response(e)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nimplementing TextFile Generator s setup method", "response": "def setup(self, context):\n    \"\"\"Implements TextFile Generator's setup method\"\"\"\n    myindex = context.get_partition_index()\n    self._files_to_consume = self._files[myindex::context.get_num_partitions()]\n    self.logger.info(\"TextFileSpout files to consume %s\" % self._files_to_consume)\n    self._lines_to_consume = self._get_next_lines()\n    self._emit_count = 0"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nadding config to the parser", "response": "def add_config(parser):\n  \"\"\" add config \"\"\"\n  # the default config path\n  default_config_path = config.get_heron_conf_dir()\n\n  parser.add_argument(\n      '--config-path',\n      metavar='(a string; path to cluster config; default: \"' + default_config_path + '\")',\n      default=os.path.join(config.get_heron_dir(), default_config_path))\n\n  return parser"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add_verbose(parser):\n  parser.add_argument(\n      '--verbose',\n      metavar='(a boolean; default: \"false\")',\n      type=bool,\n      default=False)\n  return parser", "response": "add optional verbose argument"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef add_tracker_url(parser):\n  parser.add_argument(\n      '--tracker_url',\n      metavar='(tracker url; default: \"' + DEFAULT_TRACKER_URL + '\")',\n      type=str, default=DEFAULT_TRACKER_URL)\n  return parser", "response": "add optional tracker_url argument"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef hex_escape(bin_str):\n  printable = string.ascii_letters + string.digits + string.punctuation + ' '\n  return ''.join(ch if ch in printable else r'0x{0:02x}'.format(ord(ch)) for ch in bin_str)", "response": "Hex encode a binary string"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nmake the http endpoint for the heron shell", "response": "def make_shell_endpoint(topologyInfo, instance_id):\n  \"\"\"\n  Makes the http endpoint for the heron shell\n  if shell port is present, otherwise returns None.\n  \"\"\"\n  # Format: container_<id>_<instance_id>\n  pplan = topologyInfo[\"physical_plan\"]\n  stmgrId = pplan[\"instances\"][instance_id][\"stmgrId\"]\n  host = pplan[\"stmgrs\"][stmgrId][\"host\"]\n  shell_port = pplan[\"stmgrs\"][stmgrId][\"shell_port\"]\n  return \"http://%s:%d\" % (host, shell_port)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef make_shell_logfiles_url(host, shell_port, _, instance_id=None):\n  if not shell_port:\n    return None\n  if not instance_id:\n    return \"http://%s:%d/browse/log-files\" % (host, shell_port)\n  else:\n    return \"http://%s:%d/file/log-files/%s.log.0\" % (host, shell_port, instance_id)", "response": "Make the url for log - files in heron - shell."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nmake the url for log - file data in heron - shell.", "response": "def make_shell_logfile_data_url(host, shell_port, instance_id, offset, length):\n  \"\"\"\n  Make the url for log-file data in heron-shell\n  from the info stored in stmgr.\n  \"\"\"\n  return \"http://%s:%d/filedata/log-files/%s.log.0?offset=%s&length=%s\" % \\\n    (host, shell_port, instance_id, offset, length)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_heron_tracker_dir():\n  path = \"/\".join(os.path.realpath(__file__).split('/')[:-8])\n  return normalized_class_path(path)", "response": "This will extract heron tracker directory from. pex file."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef parse_config_file(config_file):\n  expanded_config_file_path = os.path.expanduser(config_file)\n  if not os.path.lexists(expanded_config_file_path):\n    return None\n\n  configs = {}\n  # Read the configuration file\n  with open(expanded_config_file_path, 'r') as f:\n    configs = yaml.load(f)\n\n  return configs", "response": "This will parse the config file for the tracker\n  \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7 the tracker\n \u00d7"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _handle_register_response(self, response):\n    if response.status.status != common_pb2.StatusCode.Value(\"OK\"):\n      raise RuntimeError(\"Stream Manager returned a not OK response for register\")\n    Log.info(\"We registered ourselves to the Stream Manager\")\n\n    self.is_registered = True\n    if response.HasField(\"pplan\"):\n      Log.info(\"Handling assignment message from response\")\n      self._handle_assignment_message(response.pplan)\n    else:\n      Log.debug(\"Received a register response with no pplan\")", "response": "Called when a RegisterInstanceResponse arrives"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _handle_assignment_message(self, pplan):\n    Log.debug(\"In handle_assignment_message() of STStmgrClient, Physical Plan: \\n%s\", str(pplan))\n    self.heron_instance_cls.handle_assignment_msg(pplan)", "response": "Called when a new NewInstanceAssignmentMessage arrives"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef decode_packet(packet):\n    if not packet.is_complete:\n      raise RuntimeError(\"In decode_packet(): Packet corrupted\")\n\n    data = packet.data\n\n    len_typename = HeronProtocol.unpack_int(data[:4])\n    data = data[4:]\n\n    typename = data[:len_typename]\n    data = data[len_typename:]\n\n    reqid = REQID.unpack(data[:REQID.REQID_SIZE])\n    data = data[REQID.REQID_SIZE:]\n\n    len_msg = HeronProtocol.unpack_int(data[:4])\n    data = data[4:]\n\n    serialized_msg = data[:len_msg]\n\n    return typename, reqid, serialized_msg", "response": "Decodes an IncomingPacket object and returns a tuple containing the type name reqid and serialized message."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a new Outgoing Packet from a given REQID and message.", "response": "def create_packet(reqid, message):\n    \"\"\"Creates Outgoing Packet from a given reqid and message\n\n    :param reqid: REQID object\n    :param message: protocol buffer object\n    \"\"\"\n    assert message.IsInitialized()\n    packet = ''\n\n    # calculate the totla size of the packet incl. header\n    typename = message.DESCRIPTOR.full_name\n\n    datasize = HeronProtocol.get_size_to_pack_string(typename) + \\\n               REQID.REQID_SIZE + HeronProtocol.get_size_to_pack_message(message)\n\n    # first write out how much data is there as the header\n    packet += HeronProtocol.pack_int(datasize)\n\n    # next write the type string\n    packet += HeronProtocol.pack_int(len(typename))\n    packet += typename\n\n    # reqid\n    packet += reqid.pack()\n\n    # add the proto\n    packet += HeronProtocol.pack_int(message.ByteSize())\n    packet += message.SerializeToString()\n    return OutgoingPacket(packet)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef send(self, dispatcher):\n    if self.sent_complete:\n      return\n\n    sent = dispatcher.send(self.to_send)\n    self.to_send = self.to_send[sent:]", "response": "Sends this outgoing packet to dispatcher s socket"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncreate an IncomingPacket object from header and data", "response": "def create_packet(header, data):\n    \"\"\"Creates an IncomingPacket object from header and data\n\n    This method is for testing purposes\n    \"\"\"\n    packet = IncomingPacket()\n    packet.header = header\n    packet.data = data\n\n    if len(header) == HeronProtocol.HEADER_SIZE:\n      packet.is_header_read = True\n      if len(data) == packet.get_datasize():\n        packet.is_complete = True\n\n    return packet"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef read(self, dispatcher):\n    try:\n      if not self.is_header_read:\n        # try reading header\n        to_read = HeronProtocol.HEADER_SIZE - len(self.header)\n        self.header += dispatcher.recv(to_read)\n        if len(self.header) == HeronProtocol.HEADER_SIZE:\n          self.is_header_read = True\n        else:\n          Log.debug(\"Header read incomplete; read %d bytes of header\" % len(self.header))\n          return\n\n      if self.is_header_read and not self.is_complete:\n        # try reading data\n        to_read = self.get_datasize() - len(self.data)\n        self.data += dispatcher.recv(to_read)\n        if len(self.data) == self.get_datasize():\n          self.is_complete = True\n    except socket.error as e:\n      if e.errno == socket.errno.EAGAIN or e.errno == socket.errno.EWOULDBLOCK:\n        # Try again later -> call continue_read later\n        Log.debug(\"Try again error\")\n      else:\n        # Fatal error\n        Log.debug(\"Fatal error when reading IncomingPacket\")\n        raise RuntimeError(\"Fatal error occured in IncomingPacket.read()\")", "response": "Reads incoming data from asyncore. dispatcher"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef generate():\n    data_bytes = bytearray(random.getrandbits(8) for i in range(REQID.REQID_SIZE))\n    return REQID(data_bytes)", "response": "Generates a random REQID for a request"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a parser for the command", "response": "def create_parser(subparsers):\n  '''\n  :param subparsers:\n  :return:\n  '''\n  parser = subparsers.add_parser(\n      'restart',\n      help='Restart a topology',\n      usage=\"%(prog)s [options] cluster/[role]/[env] <topology-name> [container-id]\",\n      add_help=True)\n\n  args.add_titles(parser)\n  args.add_cluster_role_env(parser)\n  args.add_topology(parser)\n\n  parser.add_argument(\n      'container-id',\n      nargs='?',\n      type=int,\n      default=-1,\n      help='Identifier of the container to be restarted')\n\n  args.add_config(parser)\n  args.add_service_url(parser)\n  args.add_verbose(parser)\n\n  parser.set_defaults(subcommand='restart')\n  return parser"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_heron_config():\n  '''\n  Get config opts from the global variable\n  :return:\n  '''\n  opt_list = []\n  for (key, value) in config_opts.items():\n    opt_list.append('%s=%s' % (key, value))\n\n  all_opts = (','.join(opt_list)).replace(' ', '%%%%')\n  return all_opts", "response": "Get config opts from the global variable\n "}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef yaml_config_reader(config_path):\n  if not config_path.endswith(\".yaml\"):\n    raise ValueError(\"Config file not yaml\")\n\n  with open(config_path, 'r') as f:\n    config = yaml.load(f)\n\n  return config", "response": "Reads yaml config file and returns auto - typed config_dict"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncall when a new HeronTupleSet2 arrives in the stream", "response": "def handle_new_tuple_set_2(self, hts2):\n    \"\"\"Called when new HeronTupleSet2 arrives\n       Convert(Assemble) HeronTupleSet2(raw byte array) to HeronTupleSet\n       See more at GitHub PR #1421\n    :param tuple_msg_set: HeronTupleSet2 type\n    \"\"\"\n    if self.my_pplan_helper is None or self.my_instance is None:\n      Log.error(\"Got tuple set when no instance assigned yet\")\n    else:\n      hts = tuple_pb2.HeronTupleSet()\n      if hts2.HasField('control'):\n        hts.control.CopyFrom(hts2.control)\n      else:\n        hdts = tuple_pb2.HeronDataTupleSet()\n        hdts.stream.CopyFrom(hts2.data.stream)\n        try:\n          for trunk in hts2.data.tuples:\n            added_tuple = hdts.tuples.add()\n            added_tuple.ParseFromString(trunk)\n        except Exception:\n          Log.exception('Fail to deserialize HeronDataTuple')\n        hts.data.CopyFrom(hdts)\n      self.in_stream.offer(hts)\n      if self.my_pplan_helper.is_topology_running():\n        self.my_instance.py_class.process_incoming_tuples()"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncall when we get InitiateStatefulCheckpoint message", "response": "def handle_initiate_stateful_checkpoint(self, ckptmsg):\n    \"\"\"Called when we get InitiateStatefulCheckpoint message\n    :param ckptmsg: InitiateStatefulCheckpoint type\n    \"\"\"\n    self.in_stream.offer(ckptmsg)\n    if self.my_pplan_helper.is_topology_running():\n      self.my_instance.py_class.process_incoming_tuples()"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalls when we receive StartInstanceStatefulProcessing message", "response": "def handle_start_stateful_processing(self, start_msg):\n    \"\"\"Called when we receive StartInstanceStatefulProcessing message\n    :param start_msg: StartInstanceStatefulProcessing type\n    \"\"\"\n    Log.info(\"Received start stateful processing for %s\" % start_msg.checkpoint_id)\n    self.is_stateful_started = True\n    self.start_instance_if_possible()"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef handle_restore_instance_state(self, restore_msg):\n    Log.info(\"Restoring instance state to checkpoint %s\" % restore_msg.state.checkpoint_id)\n    # Stop the instance\n    if self.is_stateful_started:\n      self.my_instance.py_class.stop()\n      self.my_instance.py_class.clear_collector()\n      self.is_stateful_started = False\n\n    # Clear all buffers\n    self.in_stream.clear()\n    self.out_stream.clear()\n\n    # Deser the state\n    if self.stateful_state is not None:\n      self.stateful_state.clear()\n    if restore_msg.state.state is not None and restore_msg.state.state:\n      try:\n        self.stateful_state = self.serializer.deserialize(restore_msg.state.state)\n      except Exception as e:\n        raise RuntimeError(\"Could not serialize state during restore \" + str(e))\n    else:\n      Log.info(\"The restore request does not have an actual state\")\n    if self.stateful_state is None:\n      self.stateful_state = HashMapState()\n\n    Log.info(\"Instance restore state deserialized\")\n\n    # Send the response back\n    resp = ckptmgr_pb2.RestoreInstanceStateResponse()\n    resp.status.status = common_pb2.StatusCode.Value(\"OK\")\n    resp.checkpoint_id = restore_msg.state.checkpoint_id\n    self._stmgr_client.send_message(resp)", "response": "Called when we receive a RestoreInstanceStateRequest message"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsend buffered messages in out_stream to the Stream Manager", "response": "def send_buffered_messages(self):\n    \"\"\"Send messages in out_stream to the Stream Manager\"\"\"\n    while not self.out_stream.is_empty() and self._stmgr_client.is_registered:\n      tuple_set = self.out_stream.poll()\n      if isinstance(tuple_set, tuple_pb2.HeronTupleSet):\n        tuple_set.src_task_id = self.my_pplan_helper.my_task_id\n        self.gateway_metrics.update_sent_packet(tuple_set.ByteSize())\n      self._stmgr_client.send_message(tuple_set)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _handle_state_change_msg(self, new_helper):\n    assert self.my_pplan_helper is not None\n    assert self.my_instance is not None and self.my_instance.py_class is not None\n\n    if self.my_pplan_helper.get_topology_state() != new_helper.get_topology_state():\n      # handle state change\n      # update the pplan_helper\n      self.my_pplan_helper = new_helper\n      if new_helper.is_topology_running():\n        if not self.is_instance_started:\n          self.start_instance_if_possible()\n        self.my_instance.py_class.invoke_activate()\n      elif new_helper.is_topology_paused():\n        self.my_instance.py_class.invoke_deactivate()\n      else:\n        raise RuntimeError(\"Unexpected TopologyState update: %s\" % new_helper.get_topology_state())\n    else:\n      Log.info(\"Topology state remains the same.\")", "response": "Called when state change is commanded by stream manager"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncall when a NewInstanceAssignmentMessage arrives Tells this instance to become either spout or bolt.", "response": "def handle_assignment_msg(self, pplan):\n    \"\"\"Called when new NewInstanceAssignmentMessage arrives\n\n    Tells this instance to become either spout/bolt.\n\n    :param pplan: PhysicalPlan proto\n    \"\"\"\n\n    new_helper = PhysicalPlanHelper(pplan, self.instance.instance_id,\n                                    self.topo_pex_file_abs_path)\n    if self.my_pplan_helper is not None and \\\n      (self.my_pplan_helper.my_component_name != new_helper.my_component_name or\n       self.my_pplan_helper.my_task_id != new_helper.my_task_id):\n      raise RuntimeError(\"Our Assignment has changed. We will die to pick it.\")\n\n    new_helper.set_topology_context(self.metrics_collector)\n\n    if self.my_pplan_helper is None:\n      Log.info(\"Received a new Physical Plan\")\n      Log.info(\"Push the new pplan_helper to Heron Instance\")\n      self._handle_assignment_msg(new_helper)\n    else:\n      Log.info(\"Received a new Physical Plan with the same assignment -- State Change\")\n      Log.info(\"Old state: %s, new state: %s.\",\n               self.my_pplan_helper.get_topology_state(), new_helper.get_topology_state())\n      self._handle_state_change_msg(new_helper)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks if a given stream_id and tuple matches with the output schema.", "response": "def check_output_schema(self, stream_id, tup):\n    \"\"\"Checks if a given stream_id and tuple matches with the output schema\n\n    :type stream_id: str\n    :param stream_id: stream id into which tuple is sent\n    :type tup: list\n    :param tup: tuple that is going to be sent\n    \"\"\"\n    # do some checking to make sure that the number of fields match what's expected\n    size = self._output_schema.get(stream_id, None)\n    if size is None:\n      raise RuntimeError(\"%s emitting to stream %s but was not declared in output fields\"\n                         % (self.my_component_name, stream_id))\n    elif size != len(tup):\n      raise RuntimeError(\"Number of fields emitted in stream %s does not match what's expected. \"\n                         \"Expected: %s, Observed: %s\" % (stream_id, size, len(tup)))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_topology_config(self):\n    if self.pplan.topology.HasField(\"topology_config\"):\n      return self._get_dict_from_config(self.pplan.topology.topology_config)\n    else:\n      return {}", "response": "Returns the topology config"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef set_topology_context(self, metrics_collector):\n    Log.debug(\"Setting topology context\")\n    cluster_config = self.get_topology_config()\n    cluster_config.update(self._get_dict_from_config(self.my_component.config))\n    task_to_component_map = self._get_task_to_comp_map()\n    self.context = TopologyContextImpl(cluster_config, self.pplan.topology, task_to_component_map,\n                                       self.my_task_id, metrics_collector,\n                                       self.topology_pex_abs_path)", "response": "Sets a new topology context"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _get_dict_from_config(topology_config):\n    config = {}\n    for kv in topology_config.kvs:\n      if kv.HasField(\"value\"):\n        assert kv.type == topology_pb2.ConfigValueType.Value(\"STRING_VALUE\")\n        # value is string\n        if PhysicalPlanHelper._is_number(kv.value):\n          config[kv.key] = PhysicalPlanHelper._get_number(kv.value)\n        elif kv.value.lower() in (\"true\", \"false\"):\n          config[kv.key] = True if kv.value.lower() == \"true\" else False\n        else:\n          config[kv.key] = kv.value\n      elif kv.HasField(\"serialized_value\") and \\\n        kv.type == topology_pb2.ConfigValueType.Value(\"PYTHON_SERIALIZED_VALUE\"):\n        # deserialize that\n        config[kv.key] = default_serializer.deserialize(kv.serialized_value)\n      else:\n        assert kv.HasField(\"type\")\n        Log.error(\"Unsupported config <key:value> found: %s, with type: %s\"\n                  % (str(kv), str(kv.type)))\n        continue\n\n    return config", "response": "Converts Config protobuf message to python dictionary"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck whether there are any streams that consume any of my streams using custom grouping", "response": "def _setup_custom_grouping(self, topology):\n    \"\"\"Checks whether there are any bolts that consume any of my streams using custom grouping\"\"\"\n    for i in range(len(topology.bolts)):\n      for in_stream in topology.bolts[i].inputs:\n        if in_stream.stream.component_name == self.my_component_name and \\\n          in_stream.gtype == topology_pb2.Grouping.Value(\"CUSTOM\"):\n          # this bolt takes my output in custom grouping manner\n          if in_stream.type == topology_pb2.CustomGroupingObjectType.Value(\"PYTHON_OBJECT\"):\n            custom_grouping_obj = default_serializer.deserialize(in_stream.custom_grouping_object)\n            if isinstance(custom_grouping_obj, str):\n              pex_loader.load_pex(self.topology_pex_abs_path)\n              grouping_cls = \\\n                pex_loader.import_and_get_class(self.topology_pex_abs_path, custom_grouping_obj)\n              custom_grouping_obj = grouping_cls()\n            assert isinstance(custom_grouping_obj, ICustomGrouping)\n            self.custom_grouper.add(in_stream.stream.id,\n                                    self._get_taskids_for_component(topology.bolts[i].comp.name),\n                                    custom_grouping_obj,\n                                    self.my_component_name)\n\n          elif in_stream.type == topology_pb2.CustomGroupingObjectType.Value(\"JAVA_OBJECT\"):\n            raise NotImplementedError(\"Java-serialized custom grouping is not yet supported \"\n                                      \"for python topology\")\n          else:\n            raise ValueError(\"Unrecognized custom grouping type found: %s\" % str(in_stream.type))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nadd the target component to the list of tasks emitted by the given stream.", "response": "def add(self, stream_id, task_ids, grouping, source_comp_name):\n    \"\"\"Adds the target component\n\n    :type stream_id: str\n    :param stream_id: stream id into which tuples are emitted\n    :type task_ids: list of str\n    :param task_ids: list of task ids to which tuples are emitted\n    :type grouping: ICustomStreamGrouping object\n    :param grouping: custom grouping to use\n    :type source_comp_name: str\n    :param source_comp_name: source component name\n    \"\"\"\n    if stream_id not in self.targets:\n      self.targets[stream_id] = []\n    self.targets[stream_id].append(Target(task_ids, grouping, source_comp_name))"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef prepare(self, context):\n    for stream_id, targets in self.targets.items():\n      for target in targets:\n        target.prepare(context, stream_id)", "response": "Prepares the custom grouping for this component"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef choose_tasks(self, stream_id, values):\n    if stream_id not in self.targets:\n      return []\n\n    ret = []\n    for target in self.targets[stream_id]:\n      ret.extend(target.choose_tasks(values))\n    return ret", "response": "Choose tasks for a given stream_id and values and Returns a list of target tasks"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef prepare(self, context, stream_id):\n    self.grouping.prepare(context, self.source_comp_name, stream_id, self.task_ids)", "response": "Invoke prepare method of this custom grouping"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef choose_tasks(self, values):\n    ret = self.grouping.choose_tasks(values)\n    if not isinstance(ret, list):\n      raise TypeError(\"Returned object after custom grouping's choose_tasks() \"\n                      \"needs to be a list, given: %s\" % str(type(ret)))\n    else:\n      for i in ret:\n        if not isinstance(i, int):\n          raise TypeError(\"Returned object after custom grouping's choose_tasks() \"\n                          \"contained non-integer: %s\" % str(i))\n        if i not in self.task_ids:\n          raise ValueError(\"Returned object after custom grouping's choose_tasks() contained \"\n                           \"a task id that is not registered: %d\" % i)\n      return ret", "response": "Invoke the custom grouping s choose_tasks method of this custom grouping."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding the configuration file to the given parser.", "response": "def add_config(parser):\n  '''\n  :param parser:\n  :return:\n  '''\n  # the default config path\n  default_config_path = config.get_heron_conf_dir()\n\n  parser.add_argument(\n      '--config-path',\n      default=os.path.join(config.get_heron_dir(), default_config_path),\n      help='Path to cluster configuration files')\n\n  parser.add_argument(\n      '--config-property',\n      metavar='PROPERTY=VALUE',\n      action='append',\n      default=[],\n      help='Configuration properties that overrides default options')\n  return parser"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef add_dry_run(parser):\n  '''\n  :param parser:\n  :return:\n  '''\n  default_format = 'table'\n  resp_formats = ['raw', 'table', 'colored_table', 'json']\n  available_options = ', '.join(['%s' % opt for opt in resp_formats])\n\n  def dry_run_resp_format(value):\n    if value not in resp_formats:\n      raise argparse.ArgumentTypeError(\n          'Invalid dry-run response format: %s. Available formats: %s'\n          % (value, available_options))\n    return value\n\n  parser.add_argument(\n      '--dry-run',\n      default=False,\n      action='store_true',\n      help='Enable dry-run mode. Information about '\n           'the command will print but no action will be taken on the topology')\n\n  parser.add_argument(\n      '--dry-run-format',\n      metavar='DRY_RUN_FORMAT',\n      default='colored_table' if sys.stdout.isatty() else 'table',\n      type=dry_run_resp_format,\n      help='The format of the dry-run output ([%s], default=%s). '\n           'Ignored when dry-run mode is not enabled' % ('|'.join(resp_formats), default_format))\n\n  return parser", "response": "Adds a dry - run command line argument to the given parser."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread the cluster definition for server mode Read the cluster definition for server mode", "response": "def read_server_mode_cluster_definition(cluster, cl_args):\n  '''\n  Read the cluster definition for server mode\n  :param cluster:\n  :param cl_args:\n  :param config_file:\n  :return:\n  '''\n\n  client_confs = dict()\n  client_confs[cluster] = cliconfig.cluster_config(cluster)\n\n  # now check if the service-url from command line is set, if so override it\n  if cl_args.get('service_url', None):\n    client_confs[cluster]['service_url'] = cl_args['service_url']\n\n  # the return value of yaml.load can be None if conf_file is an empty file\n  # or there is no service-url in command line, if needed.\n\n  return client_confs"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nchecks the cluster definition for direct mode Check the cluster definition for direct mode", "response": "def check_direct_mode_cluster_definition(cluster, config_path):\n  '''\n  Check the cluster definition for direct mode\n  :param cluster:\n  :param config_path:\n  :return:\n  '''\n  config_path = config.get_heron_cluster_conf_dir(cluster, config_path)\n  if not os.path.isdir(config_path):\n    return False\n  return True"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef format_mode(sres):\n  mode = sres.st_mode\n\n  root = (mode & 0o700) >> 6\n  group = (mode & 0o070) >> 3\n  user = (mode & 0o7)\n\n  def stat_type(md):\n    ''' stat type'''\n    if stat.S_ISDIR(md):\n      return 'd'\n    elif stat.S_ISSOCK(md):\n      return 's'\n    else:\n      return '-'\n\n  def triple(md):\n    ''' triple '''\n    return '%c%c%c' % (\n        'r' if md & 0b100 else '-',\n        'w' if md & 0b010 else '-',\n        'x' if md & 0b001 else '-')\n\n  return ''.join([stat_type(mode), triple(root), triple(group), triple(user)])", "response": "Format a mode value in the directory list."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef format_mtime(mtime):\n  now = datetime.now()\n  dt = datetime.fromtimestamp(mtime)\n  return '%s %2d %5s' % (\n      dt.strftime('%b'), dt.day,\n      dt.year if dt.year != now.year else dt.strftime('%H:%M'))", "response": "Formats the mtime of a file to be displayed in directory listing."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef format_prefix(filename, sres):\n  try:\n    pwent = pwd.getpwuid(sres.st_uid)\n    user = pwent.pw_name\n  except KeyError:\n    user = sres.st_uid\n\n  try:\n    grent = grp.getgrgid(sres.st_gid)\n    group = grent.gr_name\n  except KeyError:\n    group = sres.st_gid\n\n  return '%s %3d %10s %10s %10d %s' % (\n      format_mode(sres),\n      sres.st_nlink,\n      user,\n      group,\n      sres.st_size,\n      format_mtime(sres.st_mtime),\n  )", "response": "Format a filename in the directory listing."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the list of files and directories in a path.", "response": "def get_listing(path):\n  \"\"\"\n  Returns the list of files and directories in a path.\n  Prepents a \"..\" (parent directory link) if path is not current dir.\n  \"\"\"\n  if path != \".\":\n    listing = sorted(['..'] + os.listdir(path))\n  else:\n    listing = sorted(os.listdir(path))\n  return listing"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading a chunk of a file from an offset upto the length.", "response": "def read_chunk(filename, offset=-1, length=-1, escape_data=False):\n  \"\"\"\n  Read a chunk of a file from an offset upto the length.\n  \"\"\"\n  try:\n    length = int(length)\n    offset = int(offset)\n  except ValueError:\n    return {}\n\n  if not os.path.isfile(filename):\n    return {}\n\n  try:\n    fstat = os.stat(filename)\n  except Exception:\n    return {}\n\n  if offset == -1:\n    offset = fstat.st_size\n\n  if length == -1:\n    length = fstat.st_size - offset\n\n  with open(filename, \"r\") as fp:\n    fp.seek(offset)\n    try:\n      data = fp.read(length)\n    except IOError:\n      return {}\n\n  if data:\n    data = _escape_data(data) if escape_data else data\n    return dict(offset=offset, length=len(data), data=data)\n\n  return dict(offset=offset, length=0)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\npiping output of prev_proc into to_cmd. Returns piped process.", "response": "def pipe(prev_proc, to_cmd):\n  \"\"\"\n  Pipes output of prev_proc into to_cmd.\n  Returns piped process\n  \"\"\"\n  stdin = None if prev_proc is None else prev_proc.stdout\n  process = subprocess.Popen(to_cmd,\n                             stdout=subprocess.PIPE,\n                             stdin=stdin)\n  if prev_proc is not None:\n    prev_proc.stdout.close() # Allow prev_proc to receive a SIGPIPE\n  return process"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef str_cmd(cmd, cwd, env):\n  process = subprocess.Popen(cmd, stdout=subprocess.PIPE,\n                             stderr=subprocess.PIPE, cwd=cwd, env=env)\n  stdout_builder, stderr_builder = proc.async_stdout_stderr_builder(process)\n  process.wait()\n  stdout, stderr = stdout_builder.result(), stderr_builder.result()\n  return {'command': ' '.join(cmd), 'stderr': stderr, 'stdout': stdout}", "response": "Runs the command and returns its stdout and stderr."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef chain(cmd_list):\n  command = ' | '.join(map(lambda x: ' '.join(x), cmd_list))\n  chained_proc = functools.reduce(pipe, [None] + cmd_list)\n  stdout_builder = proc.async_stdout_builder(chained_proc)\n  chained_proc.wait()\n  return {\n      'command': command,\n      'stdout': stdout_builder.result()\n  }", "response": "Chain commands into one command and return the final output."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_parser(subparsers):\n  metrics_parser = subparsers.add_parser(\n      'metrics',\n      help='Display info of a topology\\'s metrics',\n      usage=\"%(prog)s cluster/[role]/[env] topology-name [options]\",\n      add_help=False)\n  args.add_cluster_role_env(metrics_parser)\n  args.add_topology_name(metrics_parser)\n  args.add_verbose(metrics_parser)\n  args.add_tracker_url(metrics_parser)\n  args.add_config(metrics_parser)\n  args.add_component_name(metrics_parser)\n  metrics_parser.set_defaults(subcommand='metrics')\n\n  containers_parser = subparsers.add_parser(\n      'containers',\n      help='Display info of a topology\\'s containers metrics',\n      usage=\"%(prog)s cluster/[role]/[env] topology-name [options]\",\n      add_help=False)\n  args.add_cluster_role_env(containers_parser)\n  args.add_topology_name(containers_parser)\n  args.add_verbose(containers_parser)\n  args.add_tracker_url(containers_parser)\n  args.add_config(containers_parser)\n  args.add_container_id(containers_parser)\n  containers_parser.set_defaults(subcommand='containers')\n\n  return subparsers", "response": "create parser for topology - name"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nnormalizing raw metrics API result to table", "response": "def to_table(metrics):\n  \"\"\" normalize raw metrics API result to table \"\"\"\n  all_queries = tracker_access.metric_queries()\n  m = tracker_access.queries_map()\n  names = metrics.values()[0].keys()\n  stats = []\n  for n in names:\n    info = [n]\n    for field in all_queries:\n      try:\n        info.append(str(metrics[field][n]))\n      except KeyError:\n        pass\n    stats.append(info)\n  header = ['container id'] + [m[k] for k in all_queries if k in metrics.keys()]\n  return stats, header"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndefines the options for the network connection.", "response": "def define_options(address, port, tracker_url, base_url):\n  '''\n  :param address:\n  :param port:\n  :param tracker_url:\n  :return:\n  '''\n  define(\"address\", default=address)\n  define(\"port\", default=port)\n  define(\"tracker_url\", default=tracker_url)\n  define(\"base_url\", default=base_url)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef main():\n  '''\n  :param argv:\n  :return:\n  '''\n  log.configure(logging.DEBUG)\n  tornado.log.enable_pretty_logging()\n\n  # create the parser and parse the arguments\n  (parser, child_parser) = args.create_parsers()\n  (parsed_args, remaining) = parser.parse_known_args()\n\n  if remaining:\n    r = child_parser.parse_args(args=remaining, namespace=parsed_args)\n    namespace = vars(r)\n    if 'version' in namespace:\n      common_config.print_build_info(zipped_pex=True)\n    else:\n      parser.print_help()\n    parser.exit()\n\n  # log additional information\n  command_line_args = vars(parsed_args)\n\n  Log.info(\"Listening at http://%s:%d%s\", command_line_args['address'],\n           command_line_args['port'], command_line_args['base_url'])\n  Log.info(\"Using tracker url: %s\", command_line_args['tracker_url'])\n\n  # pass the options to tornado and start the ui server\n  define_options(command_line_args['address'],\n                 command_line_args['port'],\n                 command_line_args['tracker_url'],\n                 command_line_args['base_url'])\n  http_server = tornado.httpserver.HTTPServer(Application(command_line_args['base_url']))\n  http_server.listen(command_line_args['port'], address=command_line_args['address'])\n\n  # pylint: disable=unused-argument\n  # stop Tornado IO loop\n  def signal_handler(signum, frame):\n    # start a new line after ^C character because this looks nice\n    print('\\n', end='')\n    Log.debug('SIGINT received. Stopping UI')\n    tornado.ioloop.IOLoop.instance().stop()\n\n  # associate SIGINT and SIGTERM with a handler\n  signal.signal(signal.SIGINT, signal_handler)\n  signal.signal(signal.SIGTERM, signal_handler)\n\n  # start Tornado IO loop\n  tornado.ioloop.IOLoop.instance().start()", "response": "The main function of the command line interface."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nregister this bolt to the topology and create a HeronComponentSpec object.", "response": "def spec(cls, name=None, inputs=None, par=1, config=None, optional_outputs=None):\n    \"\"\"Register this bolt to the topology and create ``HeronComponentSpec``\n\n    This method takes an optional ``outputs`` argument for supporting dynamic output fields\n    declaration. However, it is recommended that ``outputs`` should be declared as\n    an attribute of your ``Bolt`` subclass. Also, some ways of declaring inputs is not supported\n    in this implementation; please read the documentation below.\n\n    :type name: str\n    :param name: Name of this bolt.\n    :type inputs: dict or list\n    :param inputs: Streams that feed into this Bolt.\n\n                   Two forms of this are acceptable:\n\n                   1. A `dict` mapping from ``HeronComponentSpec`` to ``Grouping``.\n                      In this case, default stream is used.\n                   2. A `dict` mapping from ``GlobalStreamId`` to ``Grouping``.\n                      This ``GlobalStreamId`` object itself is different from StreamParse, because\n                      Heron does not use thrift, although its constructor method is compatible.\n                   3. A `list` of ``HeronComponentSpec``. In this case, default stream with\n                      SHUFFLE grouping is used.\n                   4. A `list` of ``GlobalStreamId``. In this case, SHUFFLE grouping is used.\n    :type par: int\n    :param par: Parallelism hint for this spout.\n    :type config: dict\n    :param config: Component-specific config settings.\n    :type optional_outputs: list of (str or Stream) or tuple of (str or Stream)\n    :param optional_outputs: Additional output fields for this bolt. These fields are added to\n                             existing ``outputs`` class attributes of your bolt. This is an optional\n                             argument, and exists only for supporting dynamic output field\n                             declaration.\n    \"\"\"\n    python_class_path = \"%s.%s\" % (cls.__module__, cls.__name__)\n\n    if hasattr(cls, 'outputs'):\n      # avoid modification to cls.outputs\n      _outputs = copy.copy(cls.outputs)\n    else:\n      _outputs = []\n\n    if optional_outputs is not None:\n      assert isinstance(optional_outputs, (list, tuple))\n      for out in optional_outputs:\n        assert isinstance(out, (str, Stream))\n        _outputs.append(out)\n\n    return HeronComponentSpec(name, python_class_path, is_spout=False, par=par,\n                              inputs=inputs, outputs=_outputs, config=config)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef emit(self, tup, stream=Stream.DEFAULT_STREAM_ID,\n           anchors=None, direct_task=None, need_task_ids=False):\n    \"\"\"Emits a new tuple from this Bolt\n\n    It is compatible with StreamParse API.\n\n    :type tup: list or tuple\n    :param tup: the new output Tuple to send from this bolt,\n                which should contain only serializable data.\n    :type stream: str\n    :param stream: the ID of the stream to emit this Tuple to.\n                   Leave empty to emit to the default stream.\n    :type anchors: list\n    :param anchors: a list of HeronTuples to which the emitted Tuples should be anchored.\n    :type direct_task: int\n    :param direct_task: the task to send the Tuple to if performing a direct emit.\n    :type need_task_ids: bool\n    :param need_task_ids: indicate whether or not you would like the task IDs the Tuple was emitted.\n    \"\"\"\n    self.delegate.emit(tup, stream, anchors, direct_task, need_task_ids)", "response": "Emits a new tuple from this Bolt\n   "}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_time_ranges(ranges):\n  '''\n  :param ranges:\n  :return:\n  '''\n  # get the current time\n  now = int(time.time())\n\n  # form the new\n  time_slots = dict()\n\n  for key, value in ranges.items():\n    time_slots[key] = (now - value[0], now - value[1], value[2])\n\n  return (now, time_slots)", "response": "get the current time and time_slots"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding arguments to the state manager parser", "response": "def add_arguments(parser):\n  \"\"\" add arguments \"\"\"\n  default_config_file = os.path.join(\n      utils.get_heron_tracker_conf_dir(), constants.DEFAULT_CONFIG_FILE)\n\n  parser.add_argument(\n      '--config-file',\n      metavar='(a string; path to config file; default: \"' + default_config_file + '\")',\n      default=default_config_file)\n\n  parser.add_argument(\n      '--type',\n      metavar='(an string; type of state manager (zookeeper or file, etc.); example: ' \\\n        + str(constants.DEFAULT_STATE_MANAGER_TYPE) + ')',\n      choices=[\"file\", \"zookeeper\"])\n\n  parser.add_argument(\n      '--name',\n      metavar='(an string; name to be used for the state manager; example: ' \\\n        + str(constants.DEFAULT_STATE_MANAGER_NAME) + ')')\n\n  parser.add_argument(\n      '--rootpath',\n      metavar='(an string; where all the states are stored; example: ' \\\n        + str(constants.DEFAULT_STATE_MANAGER_ROOTPATH) + ')')\n\n  parser.add_argument(\n      '--tunnelhost',\n      metavar='(an string; if ssh tunneling needs to be established to connect to it; example: ' \\\n        + str(constants.DEFAULT_STATE_MANAGER_TUNNELHOST) + ')')\n\n  parser.add_argument(\n      '--hostport',\n      metavar='(an string; only used to connect to zk, must be of the form \\'host:port\\';'\\\n      ' example: ' + str(constants.DEFAULT_STATE_MANAGER_HOSTPORT) + ')')\n\n  parser.add_argument(\n      '--port',\n      metavar='(an integer; port to listen; default: ' + str(constants.DEFAULT_PORT) + ')',\n      type=int,\n      default=constants.DEFAULT_PORT)\n\n  parser.add_argument(\n      '--verbose',\n      action='store_true')\n\n  return parser"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef make_tuple(stream, tuple_key, values, roots=None):\n    component_name = stream.component_name\n    stream_id = stream.id\n    gen_task = roots[0].taskid if roots is not None and len(roots) > 0 else None\n    return HeronTuple(id=str(tuple_key), component=component_name, stream=stream_id,\n                      task=gen_task, values=values, creation_time=time.time(), roots=roots)", "response": "Creates a HeronTuple from a tuple."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a RootTupleInfo object with random bits generated from the tuple_id.", "response": "def make_root_tuple_info(stream_id, tuple_id):\n    \"\"\"Creates a RootTupleInfo\"\"\"\n    key = random.getrandbits(TupleHelper.MAX_SFIXED64_RAND_BITS)\n    return RootTupleInfo(stream_id=stream_id, tuple_id=tuple_id,\n                         insertion_time=time.time(), key=key)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfetch backpressure from the cluster.", "response": "def fetch_backpressure(self, cluster, metric, topology, component, instance, \\\n    timerange, is_max, environ=None):\n    '''\n    :param cluster:\n    :param metric:\n    :param topology:\n    :param component:\n    :param instance:\n    :param timerange:\n    :param is_max:\n    :param environ:\n    :return:\n    '''\n    pass"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ParseNolintSuppressions(filename, raw_line, linenum, error):\n  matched = Search(r'\\bNOLINT(NEXTLINE)?\\b(\\([^)]+\\))?', raw_line)\n  if matched:\n    if matched.group(1):\n      suppressed_line = linenum + 1\n    else:\n      suppressed_line = linenum\n    category = matched.group(2)\n    if category in (None, '(*)'):  # => \"suppress all\"\n      _error_suppressions.setdefault(None, set()).add(suppressed_line)\n    else:\n      if category.startswith('(') and category.endswith(')'):\n        category = category[1:-1]\n        if category in _ERROR_CATEGORIES:\n          _error_suppressions.setdefault(category, set()).add(suppressed_line)\n        elif category not in _LEGACY_ERROR_CATEGORIES:\n          error(filename, linenum, 'readability/nolint', 5,\n                'Unknown NOLINT error category: %s' % category)", "response": "Parses any NOLINT comments on the current line."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ProcessGlobalSuppresions(lines):\n  for line in lines:\n    if _SEARCH_C_FILE.search(line):\n      for category in _DEFAULT_C_SUPPRESSED_CATEGORIES:\n        _global_error_suppressions[category] = True\n    if _SEARCH_KERNEL_FILE.search(line):\n      for category in _DEFAULT_KERNEL_SUPPRESSED_CATEGORIES:\n        _global_error_suppressions[category] = True", "response": "Processes any global suppression lines in the file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn True if the specified error category is suppressed on this line.", "response": "def IsErrorSuppressedByNolint(category, linenum):\n  \"\"\"Returns true if the specified error category is suppressed on this line.\n\n  Consults the global error_suppressions map populated by\n  ParseNolintSuppressions/ProcessGlobalSuppresions/ResetNolintSuppressions.\n\n  Args:\n    category: str, the category of the error.\n    linenum: int, the current line number.\n  Returns:\n    bool, True iff the error should be suppressed due to a NOLINT comment or\n    global suppression.\n  \"\"\"\n  return (_global_error_suppressions.get(category, False) or\n          linenum in _error_suppressions.get(category, set()) or\n          linenum in _error_suppressions.get(None, set()))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef Match(pattern, s):\n  # The regexp compilation caching is inlined in both Match and Search for\n  # performance reasons; factoring it out into a separate function turns out\n  # to be noticeably expensive.\n  if pattern not in _regexp_compile_cache:\n    _regexp_compile_cache[pattern] = sre_compile.compile(pattern)\n  return _regexp_compile_cache[pattern].match(s)", "response": "Matches the string with the pattern caching the compiled regexp."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ReplaceAll(pattern, rep, s):\n  if pattern not in _regexp_compile_cache:\n    _regexp_compile_cache[pattern] = sre_compile.compile(pattern)\n  return _regexp_compile_cache[pattern].sub(rep, s)", "response": "Replaces instances of pattern in a string with a replacement."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef Search(pattern, s):\n  if pattern not in _regexp_compile_cache:\n    _regexp_compile_cache[pattern] = sre_compile.compile(pattern)\n  return _regexp_compile_cache[pattern].search(s)", "response": "Searches the string for the pattern caching the compiled regexp."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _ShouldPrintError(category, confidence, linenum):\n\n  # There are three ways we might decide not to print an error message:\n  # a \"NOLINT(category)\" comment appears in the source,\n  # the verbosity level isn't high enough, or the filters filter it out.\n  if IsErrorSuppressedByNolint(category, linenum):\n    return False\n\n  if confidence < _cpplint_state.verbose_level:\n    return False\n\n  is_filtered = False\n  for one_filter in _Filters():\n    if one_filter.startswith('-'):\n      if category.startswith(one_filter[1:]):\n        is_filtered = True\n    elif one_filter.startswith('+'):\n      if category.startswith(one_filter[1:]):\n        is_filtered = False\n    else:\n      assert False  # should have been checked for in SetFilter.\n  if is_filtered:\n    return False\n\n  return True", "response": "Returns True if the error message should be printed."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef IsCppString(line):\n\n  line = line.replace(r'\\\\', 'XX')  # after this, \\\\\" does not match to \\\"\n  return ((line.count('\"') - line.count(r'\\\"') - line.count(\"'\\\"'\")) & 1) == 1", "response": "Returns True if the next character in line is inside a C ++ string constant."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nremoving C ++ 11 raw strings from lines.", "response": "def CleanseRawStrings(raw_lines):\n  \"\"\"Removes C++11 raw strings from lines.\n\n    Before:\n      static const char kData[] = R\"(\n          multi-line string\n          )\";\n\n    After:\n      static const char kData[] = \"\"\n          (replaced by blank line)\n          \"\";\n\n  Args:\n    raw_lines: list of raw lines.\n\n  Returns:\n    list of lines with C++11 raw strings replaced by empty strings.\n  \"\"\"\n\n  delimiter = None\n  lines_without_raw_strings = []\n  for line in raw_lines:\n    if delimiter:\n      # Inside a raw string, look for the end\n      end = line.find(delimiter)\n      if end >= 0:\n        # Found the end of the string, match leading space for this\n        # line and resume copying the original lines, and also insert\n        # a \"\" on the last line.\n        leading_space = Match(r'^(\\s*)\\S', line)\n        line = leading_space.group(1) + '\"\"' + line[end + len(delimiter):]\n        delimiter = None\n      else:\n        # Haven't found the end yet, append a blank line.\n        line = '\"\"'\n\n    # Look for beginning of a raw string, and replace them with\n    # empty strings.  This is done in a loop to handle multiple raw\n    # strings on the same line.\n    while delimiter is None:\n      # Look for beginning of a raw string.\n      # See 2.14.15 [lex.string] for syntax.\n      #\n      # Once we have matched a raw string, we check the prefix of the\n      # line to make sure that the line is not part of a single line\n      # comment.  It's done this way because we remove raw strings\n      # before removing comments as opposed to removing comments\n      # before removing raw strings.  This is because there are some\n      # cpplint checks that requires the comments to be preserved, but\n      # we don't want to check comments that are inside raw strings.\n      matched = Match(r'^(.*?)\\b(?:R|u8R|uR|UR|LR)\"([^\\s\\\\()]*)\\((.*)$', line)\n      if (matched and\n          not Match(r'^([^\\'\"]|\\'(\\\\.|[^\\'])*\\'|\"(\\\\.|[^\"])*\")*//',\n                    matched.group(1))):\n        delimiter = ')' + matched.group(2) + '\"'\n\n        end = matched.group(3).find(delimiter)\n        if end >= 0:\n          # Raw string ended on same line\n          line = (matched.group(1) + '\"\"' +\n                  matched.group(3)[end + len(delimiter):])\n          delimiter = None\n        else:\n          # Start of a multi-line raw string\n          line = matched.group(1) + '\"\"'\n      else:\n        break\n\n    lines_without_raw_strings.append(line)\n\n  # TODO(unknown): if delimiter is not None here, we might want to\n  # emit a warning for unterminated string.\n  return lines_without_raw_strings"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef FindNextMultiLineCommentStart(lines, lineix):\n  while lineix < len(lines):\n    if lines[lineix].strip().startswith('/*'):\n      # Only return this marker if the comment goes beyond this line\n      if lines[lineix].strip().find('*/', 2) < 0:\n        return lineix\n    lineix += 1\n  return len(lines)", "response": "Find the beginning marker for a multiline comment."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef FindNextMultiLineCommentEnd(lines, lineix):\n  while lineix < len(lines):\n    if lines[lineix].strip().endswith('*/'):\n      return lineix\n    lineix += 1\n  return len(lines)", "response": "Find the next multi - line comment end marker."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nclears a range of lines for multi - line comments.", "response": "def RemoveMultiLineCommentsFromRange(lines, begin, end):\n  \"\"\"Clears a range of lines for multi-line comments.\"\"\"\n  # Having // dummy comments makes the lines non-empty, so we will not get\n  # unnecessary blank line warnings later in the code.\n  for i in range(begin, end):\n    lines[i] = '/**/'"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nremoves multiline comments from lines.", "response": "def RemoveMultiLineComments(filename, lines, error):\n  \"\"\"Removes multiline (c-style) comments from lines.\"\"\"\n  lineix = 0\n  while lineix < len(lines):\n    lineix_begin = FindNextMultiLineCommentStart(lines, lineix)\n    if lineix_begin >= len(lines):\n      return\n    lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin)\n    if lineix_end >= len(lines):\n      error(filename, lineix_begin + 1, 'readability/multiline_comment', 5,\n            'Could not find end of multi-line comment')\n      return\n    RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1)\n    lineix = lineix_end + 1"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nremove comments and single - line C - style comments.", "response": "def CleanseComments(line):\n  \"\"\"Removes //-comments and single-line C-style /* */ comments.\n\n  Args:\n    line: A line of C++ source.\n\n  Returns:\n    The line with single-line comments removed.\n  \"\"\"\n  commentpos = line.find('//')\n  if commentpos != -1 and not IsCppString(line[:commentpos]):\n    line = line[:commentpos].rstrip()\n  # get rid of /* ... */\n  return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfind the position just after the end of the current parenthesized expression in a line.", "response": "def FindEndOfExpressionInLine(line, startpos, stack):\n  \"\"\"Find the position just after the end of current parenthesized expression.\n\n  Args:\n    line: a CleansedLines line.\n    startpos: start searching at this position.\n    stack: nesting stack at startpos.\n\n  Returns:\n    On finding matching end: (index just after matching end, None)\n    On finding an unclosed expression: (-1, None)\n    Otherwise: (-1, new stack at end of this line)\n  \"\"\"\n  for i in xrange(startpos, len(line)):\n    char = line[i]\n    if char in '([{':\n      # Found start of parenthesized expression, push to expression stack\n      stack.append(char)\n    elif char == '<':\n      # Found potential start of template argument list\n      if i > 0 and line[i - 1] == '<':\n        # Left shift operator\n        if stack and stack[-1] == '<':\n          stack.pop()\n          if not stack:\n            return (-1, None)\n      elif i > 0 and Search(r'\\boperator\\s*$', line[0:i]):\n        # operator<, don't add to stack\n        continue\n      else:\n        # Tentative start of template argument list\n        stack.append('<')\n    elif char in ')]}':\n      # Found end of parenthesized expression.\n      #\n      # If we are currently expecting a matching '>', the pending '<'\n      # must have been an operator.  Remove them from expression stack.\n      while stack and stack[-1] == '<':\n        stack.pop()\n      if not stack:\n        return (-1, None)\n      if ((stack[-1] == '(' and char == ')') or\n          (stack[-1] == '[' and char == ']') or\n          (stack[-1] == '{' and char == '}')):\n        stack.pop()\n        if not stack:\n          return (i + 1, None)\n      else:\n        # Mismatched parentheses\n        return (-1, None)\n    elif char == '>':\n      # Found potential end of template argument list.\n\n      # Ignore \"->\" and operator functions\n      if (i > 0 and\n          (line[i - 1] == '-' or Search(r'\\boperator\\s*$', line[0:i - 1]))):\n        continue\n\n      # Pop the stack if there is a matching '<'.  Otherwise, ignore\n      # this '>' since it must be an operator.\n      if stack:\n        if stack[-1] == '<':\n          stack.pop()\n          if not stack:\n            return (i + 1, None)\n    elif char == ';':\n      # Found something that look like end of statements.  If we are currently\n      # expecting a '>', the matching '<' must have been an operator, since\n      # template argument list should not contain statements.\n      while stack and stack[-1] == '<':\n        stack.pop()\n      if not stack:\n        return (-1, None)\n\n  # Did not find end of expression or unbalanced parentheses on this line\n  return (-1, stack)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CloseExpression(clean_lines, linenum, pos):\n\n  line = clean_lines.elided[linenum]\n  if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]):\n    return (line, clean_lines.NumLines(), -1)\n\n  # Check first line\n  (end_pos, stack) = FindEndOfExpressionInLine(line, pos, [])\n  if end_pos > -1:\n    return (line, linenum, end_pos)\n\n  # Continue scanning forward\n  while stack and linenum < clean_lines.NumLines() - 1:\n    linenum += 1\n    line = clean_lines.elided[linenum]\n    (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack)\n    if end_pos > -1:\n      return (line, linenum, end_pos)\n\n  # Did not find end of expression before end of file, give up\n  return (line, clean_lines.NumLines(), -1)", "response": "This function finds the position that closes the expression."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef FindStartOfExpressionInLine(line, endpos, stack):\n  i = endpos\n  while i >= 0:\n    char = line[i]\n    if char in ')]}':\n      # Found end of expression, push to expression stack\n      stack.append(char)\n    elif char == '>':\n      # Found potential end of template argument list.\n      #\n      # Ignore it if it's a \"->\" or \">=\" or \"operator>\"\n      if (i > 0 and\n          (line[i - 1] == '-' or\n           Match(r'\\s>=\\s', line[i - 1:]) or\n           Search(r'\\boperator\\s*$', line[0:i]))):\n        i -= 1\n      else:\n        stack.append('>')\n    elif char == '<':\n      # Found potential start of template argument list\n      if i > 0 and line[i - 1] == '<':\n        # Left shift operator\n        i -= 1\n      else:\n        # If there is a matching '>', we can pop the expression stack.\n        # Otherwise, ignore this '<' since it must be an operator.\n        if stack and stack[-1] == '>':\n          stack.pop()\n          if not stack:\n            return (i, None)\n    elif char in '([{':\n      # Found start of expression.\n      #\n      # If there are any unmatched '>' on the stack, they must be\n      # operators.  Remove those.\n      while stack and stack[-1] == '>':\n        stack.pop()\n      if not stack:\n        return (-1, None)\n      if ((char == '(' and stack[-1] == ')') or\n          (char == '[' and stack[-1] == ']') or\n          (char == '{' and stack[-1] == '}')):\n        stack.pop()\n        if not stack:\n          return (i, None)\n      else:\n        # Mismatched parentheses\n        return (-1, None)\n    elif char == ';':\n      # Found something that look like end of statements.  If we are currently\n      # expecting a '<', the matching '>' must have been an operator, since\n      # template argument list should not contain statements.\n      while stack and stack[-1] == '>':\n        stack.pop()\n      if not stack:\n        return (-1, None)\n\n    i -= 1\n\n  return (-1, stack)", "response": "Find the start of the current expression in a line."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ReverseCloseExpression(clean_lines, linenum, pos):\n  line = clean_lines.elided[linenum]\n  if line[pos] not in ')}]>':\n    return (line, 0, -1)\n\n  # Check last line\n  (start_pos, stack) = FindStartOfExpressionInLine(line, pos, [])\n  if start_pos > -1:\n    return (line, linenum, start_pos)\n\n  # Continue scanning backward\n  while stack and linenum > 0:\n    linenum -= 1\n    line = clean_lines.elided[linenum]\n    (start_pos, stack) = FindStartOfExpressionInLine(line, len(line) - 1, stack)\n    if start_pos > -1:\n      return (line, linenum, start_pos)\n\n  # Did not find start of expression before beginning of file, give up\n  return (line, 0, -1)", "response": "Reverse close an expression."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CheckForCopyright(filename, lines, error):\n\n  # We'll say it should occur by line 10. Don't forget there's a\n  # dummy line at the front.\n  for line in range(1, min(len(lines), 11)):\n    if re.search(r'Copyright', lines[line], re.I): break\n  else:                       # means no copyright line was found\n    error(filename, 0, 'legal/copyright', 5,\n          'No copyright message found.  '\n          'You should have a line: \"Copyright [year] <Copyright Owner>\"')", "response": "Logs an error if no Copyright message appears at the top of the file."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef GetIndentLevel(line):\n  indent = Match(r'^( *)\\S', line)\n  if indent:\n    return len(indent.group(1))\n  else:\n    return 0", "response": "Returns the number of leading spaces in line.\n   "}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef GetHeaderGuardCPPVariable(filename):\n\n  # Restores original filename in case that cpplint is invoked from Emacs's\n  # flymake.\n  filename = re.sub(r'_flymake\\.h$', '.h', filename)\n  filename = re.sub(r'/\\.flymake/([^/]*)$', r'/\\1', filename)\n  # Replace 'c++' with 'cpp'.\n  filename = filename.replace('C++', 'cpp').replace('c++', 'cpp')\n\n  fileinfo = FileInfo(filename)\n  file_path_from_root = fileinfo.RepositoryName()\n  if _root:\n    suffix = os.sep\n    # On Windows using directory separator will leave us with\n    # \"bogus escape error\" unless we properly escape regex.\n    if suffix == '\\\\':\n      suffix += '\\\\'\n    file_path_from_root = re.sub('^' + _root + suffix, '', file_path_from_root)\n  return re.sub(r'[^a-zA-Z0-9]', '_', file_path_from_root).upper() + '_'", "response": "Returns the CPP variable that should be used as a header guard in the specified file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchecking that the file contains a header guard.", "response": "def CheckForHeaderGuard(filename, clean_lines, error):\n  \"\"\"Checks that the file contains a header guard.\n\n  Logs an error if no #ifndef header guard is present.  For other\n  headers, checks that the full pathname is used.\n\n  Args:\n    filename: The name of the C++ header file.\n    clean_lines: A CleansedLines instance containing the file.\n    error: The function to call with any errors found.\n  \"\"\"\n\n  # Don't check for header guards if there are error suppression\n  # comments somewhere in this file.\n  #\n  # Because this is silencing a warning for a nonexistent line, we\n  # only support the very specific NOLINT(build/header_guard) syntax,\n  # and not the general NOLINT or NOLINT(*) syntax.\n  raw_lines = clean_lines.lines_without_raw_strings\n  for i in raw_lines:\n    if Search(r'//\\s*NOLINT\\(build/header_guard\\)', i):\n      return\n\n  # Allow pragma once instead of header guards\n  for i in raw_lines:\n    if Search(r'^\\s*#pragma\\s+once', i):\n      return\n\n  cppvar = GetHeaderGuardCPPVariable(filename)\n\n  ifndef = ''\n  ifndef_linenum = 0\n  define = ''\n  endif = ''\n  endif_linenum = 0\n  for linenum, line in enumerate(raw_lines):\n    linesplit = line.split()\n    if len(linesplit) >= 2:\n      # find the first occurrence of #ifndef and #define, save arg\n      if not ifndef and linesplit[0] == '#ifndef':\n        # set ifndef to the header guard presented on the #ifndef line.\n        ifndef = linesplit[1]\n        ifndef_linenum = linenum\n      if not define and linesplit[0] == '#define':\n        define = linesplit[1]\n    # find the last occurrence of #endif, save entire line\n    if line.startswith('#endif'):\n      endif = line\n      endif_linenum = linenum\n\n  if not ifndef or not define or ifndef != define:\n    error(filename, 0, 'build/header_guard', 5,\n          'No #ifndef header guard found, suggested CPP variable is: %s' %\n          cppvar)\n    return\n\n  # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__\n  # for backward compatibility.\n  if ifndef != cppvar:\n    error_level = 0\n    if ifndef != cppvar + '_':\n      error_level = 5\n\n    ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], ifndef_linenum,\n                            error)\n    error(filename, ifndef_linenum, 'build/header_guard', error_level,\n          '#ifndef header guard has wrong style, please use: %s' % cppvar)\n\n  # Check for \"//\" comments on endif line.\n  ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum,\n                          error)\n  match = Match(r'#endif\\s*//\\s*' + cppvar + r'(_)?\\b', endif)\n  if match:\n    if match.group(1) == '_':\n      # Issue low severity warning for deprecated double trailing underscore\n      error(filename, endif_linenum, 'build/header_guard', 0,\n            '#endif line should be \"#endif  // %s\"' % cppvar)\n    return\n\n  # Didn't find the corresponding \"//\" comment.  If this file does not\n  # contain any \"//\" comments at all, it could be that the compiler\n  # only wants \"/**/\" comments, look for those instead.\n  no_single_line_comments = True\n  for i in xrange(1, len(raw_lines) - 1):\n    line = raw_lines[i]\n    if Match(r'^(?:(?:\\'(?:\\.|[^\\'])*\\')|(?:\"(?:\\.|[^\"])*\")|[^\\'\"])*//', line):\n      no_single_line_comments = False\n      break\n\n  if no_single_line_comments:\n    match = Match(r'#endif\\s*/\\*\\s*' + cppvar + r'(_)?\\s*\\*/', endif)\n    if match:\n      if match.group(1) == '_':\n        # Low severity warning for double trailing underscore\n        error(filename, endif_linenum, 'build/header_guard', 0,\n              '#endif line should be \"#endif  /* %s */\"' % cppvar)\n      return\n\n  # Didn't find anything\n  error(filename, endif_linenum, 'build/header_guard', 5,\n        '#endif line should be \"#endif  // %s\"' % cppvar)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CheckHeaderFileIncluded(filename, include_state, error):\n\n  # Do not check test files\n  fileinfo = FileInfo(filename)\n  if Search(_TEST_FILE_SUFFIX, fileinfo.BaseName()):\n    return\n\n  for ext in GetHeaderExtensions():\n      basefilename = filename[0:len(filename) - len(fileinfo.Extension())]\n      headerfile = basefilename + '.' + ext\n      if not os.path.exists(headerfile):\n        continue\n      headername = FileInfo(headerfile).RepositoryName()\n      first_include = None\n      for section_list in include_state.include_list:\n        for f in section_list:\n          if headername in f[0] or f[0] in headername:\n            return\n          if not first_include:\n            first_include = f[1]\n\n      error(filename, first_include, 'build/include', 5,\n            '%s should include its header file %s' % (fileinfo.RepositoryName(),\n                                                      headername))", "response": "Logs an error if a source file does not include its header."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef CheckForBadCharacters(filename, lines, error):\n  for linenum, line in enumerate(lines):\n    if unicode_escape_decode('\\ufffd') in line:\n      error(filename, linenum, 'readability/utf8', 5,\n            'Line contains invalid UTF-8 (or Unicode replacement character).')\n    if '\\0' in line:\n      error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.')", "response": "Logs an error for each line containing bad characters."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nlogging an error if there is no newline character at the end of the file.", "response": "def CheckForNewlineAtEOF(filename, lines, error):\n  \"\"\"Logs an error if there is no newline char at the end of the file.\n\n  Args:\n    filename: The name of the current file.\n    lines: An array of strings, each representing a line of the file.\n    error: The function to call with any errors found.\n  \"\"\"\n\n  # The array lines() was created by adding two newlines to the\n  # original file (go figure), then splitting on \\n.\n  # To verify that the file ends in \\n, we just have to make sure the\n  # last-but-two element of lines() exists and is empty.\n  if len(lines) < 3 or lines[-2]:\n    error(filename, len(lines) - 2, 'whitespace/ending_newline', 5,\n          'Could not find a newline character at the end of the file.')"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error):\n  line = clean_lines.elided[linenum]\n\n  # Remove all \\\\ (escaped backslashes) from the line. They are OK, and the\n  # second (escaped) slash may trigger later \\\" detection erroneously.\n  line = line.replace('\\\\\\\\', '')\n\n  if line.count('/*') > line.count('*/'):\n    error(filename, linenum, 'readability/multiline_comment', 5,\n          'Complex multi-line /*...*/-style comment found. '\n          'Lint may give bogus warnings.  '\n          'Consider replacing these with //-style comments, '\n          'with #if 0...#endif, '\n          'or with more clearly structured multi-line comments.')\n\n  if (line.count('\"') - line.count('\\\\\"')) % 2:\n    error(filename, linenum, 'readability/multiline_string', 5,\n          'Multi-line string (\"...\") found.  This lint script doesn\\'t '\n          'do well with such strings, and may give bogus warnings.  '\n          'Use C++11 raw strings or concatenation instead.')", "response": "Checks for multiline comments and strings."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CheckPosixThreading(filename, clean_lines, linenum, error):\n  line = clean_lines.elided[linenum]\n  for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST:\n    # Additional pattern matching check to confirm that this is the\n    # function we are looking for\n    if Search(pattern, line):\n      error(filename, linenum, 'runtime/threadsafe_fn', 2,\n            'Consider using ' + multithread_safe_func +\n            '...) instead of ' + single_thread_func +\n            '...) for improved thread safety.')", "response": "Checks for calls to thread - safe functions."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncheck that VLOG is only used for defining a severity level.", "response": "def CheckVlogArguments(filename, clean_lines, linenum, error):\n  \"\"\"Checks that VLOG() is only used for defining a logging level.\n\n  For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and\n  VLOG(FATAL) are not.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    error: The function to call with any errors found.\n  \"\"\"\n  line = clean_lines.elided[linenum]\n  if Search(r'\\bVLOG\\((INFO|ERROR|WARNING|DFATAL|FATAL)\\)', line):\n    error(filename, linenum, 'runtime/vlog', 5,\n          'VLOG() should be used with numeric verbosity level.  '\n          'Use LOG() if you want symbolic severity levels.')"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncheck for invalid increment.", "response": "def CheckInvalidIncrement(filename, clean_lines, linenum, error):\n  \"\"\"Checks for invalid increment *count++.\n\n  For example following function:\n  void increment_counter(int* count) {\n    *count++;\n  }\n  is invalid, because it effectively does count++, moving pointer, and should\n  be replaced with ++*count, (*count)++ or *count += 1.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    error: The function to call with any errors found.\n  \"\"\"\n  line = clean_lines.elided[linenum]\n  if _RE_PATTERN_INVALID_INCREMENT.match(line):\n    error(filename, linenum, 'runtime/invalid_increment', 5,\n          'Changing pointer instead of value (or unused value of operator*).')"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking if the correctness of various spacing around function calls.", "response": "def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error):\n  \"\"\"Checks for the correctness of various spacing around function calls.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    error: The function to call with any errors found.\n  \"\"\"\n  line = clean_lines.elided[linenum]\n\n  # Since function calls often occur inside if/for/while/switch\n  # expressions - which have their own, more liberal conventions - we\n  # first see if we should be looking inside such an expression for a\n  # function call, to which we can apply more strict standards.\n  fncall = line    # if there's no control flow construct, look at whole line\n  for pattern in (r'\\bif\\s*\\((.*)\\)\\s*{',\n                  r'\\bfor\\s*\\((.*)\\)\\s*{',\n                  r'\\bwhile\\s*\\((.*)\\)\\s*[{;]',\n                  r'\\bswitch\\s*\\((.*)\\)\\s*{'):\n    match = Search(pattern, line)\n    if match:\n      fncall = match.group(1)    # look inside the parens for function calls\n      break\n\n  # Except in if/for/while/switch, there should never be space\n  # immediately inside parens (eg \"f( 3, 4 )\").  We make an exception\n  # for nested parens ( (a+b) + c ).  Likewise, there should never be\n  # a space before a ( when it's a function argument.  I assume it's a\n  # function argument when the char before the whitespace is legal in\n  # a function name (alnum + _) and we're not starting a macro. Also ignore\n  # pointers and references to arrays and functions coz they're too tricky:\n  # we use a very simple way to recognize these:\n  # \" (something)(maybe-something)\" or\n  # \" (something)(maybe-something,\" or\n  # \" (something)[something]\"\n  # Note that we assume the contents of [] to be short enough that\n  # they'll never need to wrap.\n  if (  # Ignore control structures.\n      not Search(r'\\b(if|for|while|switch|return|new|delete|catch|sizeof)\\b',\n                 fncall) and\n      # Ignore pointers/references to functions.\n      not Search(r' \\([^)]+\\)\\([^)]*(\\)|,$)', fncall) and\n      # Ignore pointers/references to arrays.\n      not Search(r' \\([^)]+\\)\\[[^\\]]+\\]', fncall)):\n    if Search(r'\\w\\s*\\(\\s(?!\\s*\\\\$)', fncall):      # a ( used for a fn call\n      error(filename, linenum, 'whitespace/parens', 4,\n            'Extra space after ( in function call')\n    elif Search(r'\\(\\s+(?!(\\s*\\\\)|\\()', fncall):\n      error(filename, linenum, 'whitespace/parens', 2,\n            'Extra space after (')\n    if (Search(r'\\w\\s+\\(', fncall) and\n        not Search(r'_{0,2}asm_{0,2}\\s+_{0,2}volatile_{0,2}\\s+\\(', fncall) and\n        not Search(r'#\\s*define|typedef|using\\s+\\w+\\s*=', fncall) and\n        not Search(r'\\w\\s+\\((\\w+::)*\\*\\w+\\)\\(', fncall) and\n        not Search(r'\\bcase\\s+\\(', fncall)):\n      # TODO(unknown): Space after an operator function seem to be a common\n      # error, silence those for now by restricting them to highest verbosity.\n      if Search(r'\\boperator_*\\b', line):\n        error(filename, linenum, 'whitespace/parens', 0,\n              'Extra space before ( in function call')\n      else:\n        error(filename, linenum, 'whitespace/parens', 4,\n              'Extra space before ( in function call')\n    # If the ) is followed only by a newline or a { + newline, assume it's\n    # part of a control statement (if/while/etc), and don't complain\n    if Search(r'[^)]\\s+\\)\\s*[^{\\s]', fncall):\n      # If the closing parenthesis is preceded by only whitespaces,\n      # try to give a more descriptive error message.\n      if Search(r'^\\s+\\)', fncall):\n        error(filename, linenum, 'whitespace/parens', 2,\n              'Closing ) should be moved to the previous line')\n      else:\n        error(filename, linenum, 'whitespace/parens', 2,\n              'Extra space before )')"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CheckForFunctionLengths(filename, clean_lines, linenum,\n                            function_state, error):\n  \"\"\"Reports for long function bodies.\n\n  For an overview why this is done, see:\n  https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions\n\n  Uses a simplistic algorithm assuming other style guidelines\n  (especially spacing) are followed.\n  Only checks unindented functions, so class members are unchecked.\n  Trivial bodies are unchecked, so constructors with huge initializer lists\n  may be missed.\n  Blank/comment lines are not counted so as to avoid encouraging the removal\n  of vertical space and comments just to get through a lint check.\n  NOLINT *on the last line of a function* disables this check.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    function_state: Current function name and lines in body so far.\n    error: The function to call with any errors found.\n  \"\"\"\n  lines = clean_lines.lines\n  line = lines[linenum]\n  joined_line = ''\n\n  starting_func = False\n  regexp = r'(\\w(\\w|::|\\*|\\&|\\s)*)\\('  # decls * & space::name( ...\n  match_result = Match(regexp, line)\n  if match_result:\n    # If the name is all caps and underscores, figure it's a macro and\n    # ignore it, unless it's TEST or TEST_F.\n    function_name = match_result.group(1).split()[-1]\n    if function_name == 'TEST' or function_name == 'TEST_F' or (\n        not Match(r'[A-Z_]+$', function_name)):\n      starting_func = True\n\n  if starting_func:\n    body_found = False\n    for start_linenum in range(linenum, clean_lines.NumLines()):\n      start_line = lines[start_linenum]\n      joined_line += ' ' + start_line.lstrip()\n      if Search(r'(;|})', start_line):  # Declarations and trivial functions\n        body_found = True\n        break                              # ... ignore\n      elif Search(r'{', start_line):\n        body_found = True\n        function = Search(r'((\\w|:)*)\\(', line).group(1)\n        if Match(r'TEST', function):    # Handle TEST... macros\n          parameter_regexp = Search(r'(\\(.*\\))', joined_line)\n          if parameter_regexp:             # Ignore bad syntax\n            function += parameter_regexp.group(1)\n        else:\n          function += '()'\n        function_state.Begin(function)\n        break\n    if not body_found:\n      # No body for the function (or evidence of a non-function) was found.\n      error(filename, linenum, 'readability/fn_size', 5,\n            'Lint failed to find start of function body.')\n  elif Match(r'^\\}\\s*$', line):  # function end\n    function_state.Check(error, filename, linenum)\n    function_state.End()\n  elif not Match(r'^\\s*$', line):\n    function_state.Count()", "response": "Checks if the function length of the current file is well - formed."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncheck for common mistakes in comments.", "response": "def CheckComment(line, filename, linenum, next_line_start, error):\n  \"\"\"Checks for common mistakes in comments.\n\n  Args:\n    line: The line in question.\n    filename: The name of the current file.\n    linenum: The number of the line to check.\n    next_line_start: The first non-whitespace column of the next line.\n    error: The function to call with any errors found.\n  \"\"\"\n  commentpos = line.find('//')\n  if commentpos != -1:\n    # Check if the // may be in quotes.  If so, ignore it\n    if re.sub(r'\\\\.', '', line[0:commentpos]).count('\"') % 2 == 0:\n      # Allow one space for new scopes, two spaces otherwise:\n      if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and\n          ((commentpos >= 1 and\n            line[commentpos-1] not in string.whitespace) or\n           (commentpos >= 2 and\n            line[commentpos-2] not in string.whitespace))):\n        error(filename, linenum, 'whitespace/comments', 2,\n              'At least two spaces is best between code and comments')\n\n      # Checks for common mistakes in TODO comments.\n      comment = line[commentpos:]\n      match = _RE_PATTERN_TODO.match(comment)\n      if match:\n        # One whitespace is correct; zero whitespace is handled elsewhere.\n        leading_whitespace = match.group(1)\n        if len(leading_whitespace) > 1:\n          error(filename, linenum, 'whitespace/todo', 2,\n                'Too many spaces before TODO')\n\n        username = match.group(2)\n        if not username:\n          error(filename, linenum, 'readability/todo', 2,\n                'Missing username in TODO; it should look like '\n                '\"// TODO(my_username): Stuff.\"')\n\n        middle_whitespace = match.group(3)\n        # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison\n        if middle_whitespace != ' ' and middle_whitespace != '':\n          error(filename, linenum, 'whitespace/todo', 2,\n                'TODO(my_username) should be followed by a space')\n\n      # If the comment contains an alphanumeric character, there\n      # should be a space somewhere between it and the // unless\n      # it's a /// or //! Doxygen comment.\n      if (Match(r'//[^ ]*\\w', comment) and\n          not Match(r'(///|//\\!)(\\s+|$)', comment)):\n        error(filename, linenum, 'whitespace/comments', 4,\n              'Should have a space between // and comment')"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CheckAccess(filename, clean_lines, linenum, nesting_state, error):\n  line = clean_lines.elided[linenum]  # get rid of comments and strings\n\n  matched = Match((r'\\s*(DISALLOW_COPY_AND_ASSIGN|'\n                   r'DISALLOW_IMPLICIT_CONSTRUCTORS)'), line)\n  if not matched:\n    return\n  if nesting_state.stack and isinstance(nesting_state.stack[-1], _ClassInfo):\n    if nesting_state.stack[-1].access != 'private':\n      error(filename, linenum, 'readability/constructors', 3,\n            '%s must be in the private: section' % matched.group(1))\n\n  else:\n    # Found DISALLOW* macro outside a class declaration, or perhaps it\n    # was used inside a function when it should have been part of the\n    # class declaration.  We could issue a warning here, but it\n    # probably resulted in a compiler error already.\n    pass", "response": "Checks for improper use of DISALLOW * macros."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncheck for various spacing issues in the code.", "response": "def CheckSpacing(filename, clean_lines, linenum, nesting_state, error):\n  \"\"\"Checks for the correctness of various spacing issues in the code.\n\n  Things we check for: spaces around operators, spaces after\n  if/for/while/switch, no spaces around parens in function calls, two\n  spaces between code and comment, don't start a block with a blank\n  line, don't end a function with a blank line, don't add a blank line\n  after public/protected/private, don't have too many blank lines in a row.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    nesting_state: A NestingState instance which maintains information about\n                   the current stack of nested blocks being parsed.\n    error: The function to call with any errors found.\n  \"\"\"\n\n  # Don't use \"elided\" lines here, otherwise we can't check commented lines.\n  # Don't want to use \"raw\" either, because we don't want to check inside C++11\n  # raw strings,\n  raw = clean_lines.lines_without_raw_strings\n  line = raw[linenum]\n\n  # Before nixing comments, check if the line is blank for no good\n  # reason.  This includes the first line after a block is opened, and\n  # blank lines at the end of a function (ie, right before a line like '}'\n  #\n  # Skip all the blank line checks if we are immediately inside a\n  # namespace body.  In other words, don't issue blank line warnings\n  # for this block:\n  #   namespace {\n  #\n  #   }\n  #\n  # A warning about missing end of namespace comments will be issued instead.\n  #\n  # Also skip blank line checks for 'extern \"C\"' blocks, which are formatted\n  # like namespaces.\n  if (IsBlankLine(line) and\n      not nesting_state.InNamespaceBody() and\n      not nesting_state.InExternC()):\n    elided = clean_lines.elided\n    prev_line = elided[linenum - 1]\n    prevbrace = prev_line.rfind('{')\n    # TODO(unknown): Don't complain if line before blank line, and line after,\n    #                both start with alnums and are indented the same amount.\n    #                This ignores whitespace at the start of a namespace block\n    #                because those are not usually indented.\n    if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1:\n      # OK, we have a blank line at the start of a code block.  Before we\n      # complain, we check if it is an exception to the rule: The previous\n      # non-empty line has the parameters of a function header that are indented\n      # 4 spaces (because they did not fit in a 80 column line when placed on\n      # the same line as the function name).  We also check for the case where\n      # the previous line is indented 6 spaces, which may happen when the\n      # initializers of a constructor do not fit into a 80 column line.\n      exception = False\n      if Match(r' {6}\\w', prev_line):  # Initializer list?\n        # We are looking for the opening column of initializer list, which\n        # should be indented 4 spaces to cause 6 space indentation afterwards.\n        search_position = linenum-2\n        while (search_position >= 0\n               and Match(r' {6}\\w', elided[search_position])):\n          search_position -= 1\n        exception = (search_position >= 0\n                     and elided[search_position][:5] == '    :')\n      else:\n        # Search for the function arguments or an initializer list.  We use a\n        # simple heuristic here: If the line is indented 4 spaces; and we have a\n        # closing paren, without the opening paren, followed by an opening brace\n        # or colon (for initializer lists) we assume that it is the last line of\n        # a function header.  If we have a colon indented 4 spaces, it is an\n        # initializer list.\n        exception = (Match(r' {4}\\w[^\\(]*\\)\\s*(const\\s*)?(\\{\\s*$|:)',\n                           prev_line)\n                     or Match(r' {4}:', prev_line))\n\n      if not exception:\n        error(filename, linenum, 'whitespace/blank_line', 2,\n              'Redundant blank line at the start of a code block '\n              'should be deleted.')\n    # Ignore blank lines at the end of a block in a long if-else\n    # chain, like this:\n    #   if (condition1) {\n    #     // Something followed by a blank line\n    #\n    #   } else if (condition2) {\n    #     // Something else\n    #   }\n    if linenum + 1 < clean_lines.NumLines():\n      next_line = raw[linenum + 1]\n      if (next_line\n          and Match(r'\\s*}', next_line)\n          and next_line.find('} else ') == -1):\n        error(filename, linenum, 'whitespace/blank_line', 3,\n              'Redundant blank line at the end of a code block '\n              'should be deleted.')\n\n    matched = Match(r'\\s*(public|protected|private):', prev_line)\n    if matched:\n      error(filename, linenum, 'whitespace/blank_line', 3,\n            'Do not leave a blank line after \"%s:\"' % matched.group(1))\n\n  # Next, check comments\n  next_line_start = 0\n  if linenum + 1 < clean_lines.NumLines():\n    next_line = raw[linenum + 1]\n    next_line_start = len(next_line) - len(next_line.lstrip())\n  CheckComment(line, filename, linenum, next_line_start, error)\n\n  # get rid of comments and strings\n  line = clean_lines.elided[linenum]\n\n  # You shouldn't have spaces before your brackets, except maybe after\n  # 'delete []' or 'return []() {};'\n  if Search(r'\\w\\s+\\[', line) and not Search(r'(?:delete|return)\\s+\\[', line):\n    error(filename, linenum, 'whitespace/braces', 5,\n          'Extra space before [')\n\n  # In range-based for, we wanted spaces before and after the colon, but\n  # not around \"::\" tokens that might appear.\n  if (Search(r'for *\\(.*[^:]:[^: ]', line) or\n      Search(r'for *\\(.*[^: ]:[^:]', line)):\n    error(filename, linenum, 'whitespace/forcolon', 2,\n          'Missing space around colon in range-based for loop')"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CheckParenthesisSpacing(filename, clean_lines, linenum, error):\n  line = clean_lines.elided[linenum]\n\n  # No spaces after an if, while, switch, or for\n  match = Search(r' (if\\(|for\\(|while\\(|switch\\()', line)\n  if match:\n    error(filename, linenum, 'whitespace/parens', 5,\n          'Missing space before ( in %s' % match.group(1))\n\n  # For if/for/while/switch, the left and right parens should be\n  # consistent about how many spaces are inside the parens, and\n  # there should either be zero or one spaces inside the parens.\n  # We don't want: \"if ( foo)\" or \"if ( foo   )\".\n  # Exception: \"for ( ; foo; bar)\" and \"for (foo; bar; )\" are allowed.\n  match = Search(r'\\b(if|for|while|switch)\\s*'\n                 r'\\(([ ]*)(.).*[^ ]+([ ]*)\\)\\s*{\\s*$',\n                 line)\n  if match:\n    if len(match.group(2)) != len(match.group(4)):\n      if not (match.group(3) == ';' and\n              len(match.group(2)) == 1 + len(match.group(4)) or\n              not match.group(2) and Search(r'\\bfor\\s*\\(.*; \\)', line)):\n        error(filename, linenum, 'whitespace/parens', 5,\n              'Mismatching spaces inside () in %s' % match.group(1))\n    if len(match.group(2)) not in [0, 1]:\n      error(filename, linenum, 'whitespace/parens', 5,\n            'Should have zero or one spaces inside ( and ) in %s' %\n            match.group(1))", "response": "Checks for horizontal spacing around parentheses."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CheckCommaSpacing(filename, clean_lines, linenum, error):\n  raw = clean_lines.lines_without_raw_strings\n  line = clean_lines.elided[linenum]\n\n  # You should always have a space after a comma (either as fn arg or operator)\n  #\n  # This does not apply when the non-space character following the\n  # comma is another comma, since the only time when that happens is\n  # for empty macro arguments.\n  #\n  # We run this check in two passes: first pass on elided lines to\n  # verify that lines contain missing whitespaces, second pass on raw\n  # lines to confirm that those missing whitespaces are not due to\n  # elided comments.\n  if (Search(r',[^,\\s]', ReplaceAll(r'\\boperator\\s*,\\s*\\(', 'F(', line)) and\n      Search(r',[^,\\s]', raw[linenum])):\n    error(filename, linenum, 'whitespace/comma', 3,\n          'Missing space after ,')\n\n  # You should always have a space after a semicolon\n  # except for few corner cases\n  # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more\n  # space after ;\n  if Search(r';[^\\s};\\\\)/]', line):\n    error(filename, linenum, 'whitespace/semicolon', 3,\n          'Missing space after ;')", "response": "Checks for horizontal spacing near commas and semicolons."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking if the expression looks like a type name.", "response": "def _IsType(clean_lines, nesting_state, expr):\n  \"\"\"Check if expression looks like a type name, returns true if so.\n\n  Args:\n    clean_lines: A CleansedLines instance containing the file.\n    nesting_state: A NestingState instance which maintains information about\n                   the current stack of nested blocks being parsed.\n    expr: The expression to check.\n  Returns:\n    True, if token looks like a type.\n  \"\"\"\n  # Keep only the last token in the expression\n  last_word = Match(r'^.*(\\b\\S+)$', expr)\n  if last_word:\n    token = last_word.group(1)\n  else:\n    token = expr\n\n  # Match native types and stdint types\n  if _TYPES.match(token):\n    return True\n\n  # Try a bit harder to match templated types.  Walk up the nesting\n  # stack until we find something that resembles a typename\n  # declaration for what we are looking for.\n  typename_pattern = (r'\\b(?:typename|class|struct)\\s+' + re.escape(token) +\n                      r'\\b')\n  block_index = len(nesting_state.stack) - 1\n  while block_index >= 0:\n    if isinstance(nesting_state.stack[block_index], _NamespaceInfo):\n      return False\n\n    # Found where the opening brace is.  We want to scan from this\n    # line up to the beginning of the function, minus a few lines.\n    #   template <typename Type1,  // stop scanning here\n    #             ...>\n    #   class C\n    #     : public ... {  // start scanning here\n    last_line = nesting_state.stack[block_index].starting_linenum\n\n    next_block_start = 0\n    if block_index > 0:\n      next_block_start = nesting_state.stack[block_index - 1].starting_linenum\n    first_line = last_line\n    while first_line >= next_block_start:\n      if clean_lines.elided[first_line].find('template') >= 0:\n        break\n      first_line -= 1\n    if first_line < next_block_start:\n      # Didn't find any \"template\" keyword before reaching the next block,\n      # there are probably no template things to check for this block\n      block_index -= 1\n      continue\n\n    # Look for typename in the specified range\n    for i in xrange(first_line, last_line + 1, 1):\n      if Search(typename_pattern, clean_lines.elided[i]):\n        return True\n    block_index -= 1\n\n  return False"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck for horizontal spacing near commas.", "response": "def CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error):\n  \"\"\"Checks for horizontal spacing near commas.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    nesting_state: A NestingState instance which maintains information about\n                   the current stack of nested blocks being parsed.\n    error: The function to call with any errors found.\n  \"\"\"\n  line = clean_lines.elided[linenum]\n\n  # Except after an opening paren, or after another opening brace (in case of\n  # an initializer list, for instance), you should have spaces before your\n  # braces when they are delimiting blocks, classes, namespaces etc.\n  # And since you should never have braces at the beginning of a line,\n  # this is an easy test.  Except that braces used for initialization don't\n  # follow the same rule; we often don't want spaces before those.\n  match = Match(r'^(.*[^ ({>]){', line)\n\n  if match:\n    # Try a bit harder to check for brace initialization.  This\n    # happens in one of the following forms:\n    #   Constructor() : initializer_list_{} { ... }\n    #   Constructor{}.MemberFunction()\n    #   Type variable{};\n    #   FunctionCall(type{}, ...);\n    #   LastArgument(..., type{});\n    #   LOG(INFO) << type{} << \" ...\";\n    #   map_of_type[{...}] = ...;\n    #   ternary = expr ? new type{} : nullptr;\n    #   OuterTemplate<InnerTemplateConstructor<Type>{}>\n    #\n    # We check for the character following the closing brace, and\n    # silence the warning if it's one of those listed above, i.e.\n    # \"{.;,)<>]:\".\n    #\n    # To account for nested initializer list, we allow any number of\n    # closing braces up to \"{;,)<\".  We can't simply silence the\n    # warning on first sight of closing brace, because that would\n    # cause false negatives for things that are not initializer lists.\n    #   Silence this:         But not this:\n    #     Outer{                if (...) {\n    #       Inner{...}            if (...){  // Missing space before {\n    #     };                    }\n    #\n    # There is a false negative with this approach if people inserted\n    # spurious semicolons, e.g. \"if (cond){};\", but we will catch the\n    # spurious semicolon with a separate check.\n    leading_text = match.group(1)\n    (endline, endlinenum, endpos) = CloseExpression(\n        clean_lines, linenum, len(match.group(1)))\n    trailing_text = ''\n    if endpos > -1:\n      trailing_text = endline[endpos:]\n    for offset in xrange(endlinenum + 1,\n                         min(endlinenum + 3, clean_lines.NumLines() - 1)):\n      trailing_text += clean_lines.elided[offset]\n    # We also suppress warnings for `uint64_t{expression}` etc., as the style\n    # guide recommends brace initialization for integral types to avoid\n    # overflow/truncation.\n    if (not Match(r'^[\\s}]*[{.;,)<>\\]:]', trailing_text)\n        and not _IsType(clean_lines, nesting_state, leading_text)):\n      error(filename, linenum, 'whitespace/braces', 5,\n            'Missing space before {')\n\n  # Make sure '} else {' has spaces.\n  if Search(r'}else', line):\n    error(filename, linenum, 'whitespace/braces', 5,\n          'Missing space before else')\n\n  # You shouldn't have a space before a semicolon at the end of the line.\n  # There's a special case for \"for\" since the style guide allows space before\n  # the semicolon there.\n  if Search(r':\\s*;\\s*$', line):\n    error(filename, linenum, 'whitespace/semicolon', 5,\n          'Semicolon defining empty statement. Use {} instead.')\n  elif Search(r'^\\s*;\\s*$', line):\n    error(filename, linenum, 'whitespace/semicolon', 5,\n          'Line contains only semicolon. If this should be an empty statement, '\n          'use {} instead.')\n  elif (Search(r'\\s+;\\s*$', line) and\n        not Search(r'\\bfor\\b', line)):\n    error(filename, linenum, 'whitespace/semicolon', 5,\n          'Extra space before last semicolon. If this should be an empty '\n          'statement, use {} instead.')"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nchecks if the token ending on ( linenum column is decltype.", "response": "def IsDecltype(clean_lines, linenum, column):\n  \"\"\"Check if the token ending on (linenum, column) is decltype().\n\n  Args:\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: the number of the line to check.\n    column: end column of the token to check.\n  Returns:\n    True if this token is decltype() expression, False otherwise.\n  \"\"\"\n  (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column)\n  if start_col < 0:\n    return False\n  if Search(r'\\bdecltype\\s*$', text[0:start_col]):\n    return True\n  return False"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef CheckSectionSpacing(filename, clean_lines, class_info, linenum, error):\n  # Skip checks if the class is small, where small means 25 lines or less.\n  # 25 lines seems like a good cutoff since that's the usual height of\n  # terminals, and any class that can't fit in one screen can't really\n  # be considered \"small\".\n  #\n  # Also skip checks if we are on the first line.  This accounts for\n  # classes that look like\n  #   class Foo { public: ... };\n  #\n  # If we didn't find the end of the class, last_line would be zero,\n  # and the check will be skipped by the first condition.\n  if (class_info.last_line - class_info.starting_linenum <= 24 or\n      linenum <= class_info.starting_linenum):\n    return\n\n  matched = Match(r'\\s*(public|protected|private):', clean_lines.lines[linenum])\n  if matched:\n    # Issue warning if the line before public/protected/private was\n    # not a blank line, but don't do this if the previous line contains\n    # \"class\" or \"struct\".  This can happen two ways:\n    #  - We are at the beginning of the class.\n    #  - We are forward-declaring an inner class that is semantically\n    #    private, but needed to be public for implementation reasons.\n    # Also ignores cases where the previous line ends with a backslash as can be\n    # common when defining classes in C macros.\n    prev_line = clean_lines.lines[linenum - 1]\n    if (not IsBlankLine(prev_line) and\n        not Search(r'\\b(class|struct)\\b', prev_line) and\n        not Search(r'\\\\$', prev_line)):\n      # Try a bit harder to find the beginning of the class.  This is to\n      # account for multi-line base-specifier lists, e.g.:\n      #   class Derived\n      #       : public Base {\n      end_class_head = class_info.starting_linenum\n      for i in range(class_info.starting_linenum, linenum):\n        if Search(r'\\{\\s*$', clean_lines.lines[i]):\n          end_class_head = i\n          break\n      if end_class_head < linenum - 1:\n        error(filename, linenum, 'whitespace/blank_line', 3,\n              '\"%s:\" should be preceded by a blank line' % matched.group(1))", "response": "Checks for additional blank line issues related to sections."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef GetPreviousNonBlankLine(clean_lines, linenum):\n\n  prevlinenum = linenum - 1\n  while prevlinenum >= 0:\n    prevline = clean_lines.elided[prevlinenum]\n    if not IsBlankLine(prevline):     # if not a blank line...\n      return (prevline, prevlinenum)\n    prevlinenum -= 1\n  return ('', -1)", "response": "Returns the most recent non - blank line and its line number."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CheckTrailingSemicolon(filename, clean_lines, linenum, error):\n\n  line = clean_lines.elided[linenum]\n\n  # Block bodies should not be followed by a semicolon.  Due to C++11\n  # brace initialization, there are more places where semicolons are\n  # required than not, so we use a whitelist approach to check these\n  # rather than a blacklist.  These are the places where \"};\" should\n  # be replaced by just \"}\":\n  # 1. Some flavor of block following closing parenthesis:\n  #    for (;;) {};\n  #    while (...) {};\n  #    switch (...) {};\n  #    Function(...) {};\n  #    if (...) {};\n  #    if (...) else if (...) {};\n  #\n  # 2. else block:\n  #    if (...) else {};\n  #\n  # 3. const member function:\n  #    Function(...) const {};\n  #\n  # 4. Block following some statement:\n  #    x = 42;\n  #    {};\n  #\n  # 5. Block at the beginning of a function:\n  #    Function(...) {\n  #      {};\n  #    }\n  #\n  #    Note that naively checking for the preceding \"{\" will also match\n  #    braces inside multi-dimensional arrays, but this is fine since\n  #    that expression will not contain semicolons.\n  #\n  # 6. Block following another block:\n  #    while (true) {}\n  #    {};\n  #\n  # 7. End of namespaces:\n  #    namespace {};\n  #\n  #    These semicolons seems far more common than other kinds of\n  #    redundant semicolons, possibly due to people converting classes\n  #    to namespaces.  For now we do not warn for this case.\n  #\n  # Try matching case 1 first.\n  match = Match(r'^(.*\\)\\s*)\\{', line)\n  if match:\n    # Matched closing parenthesis (case 1).  Check the token before the\n    # matching opening parenthesis, and don't warn if it looks like a\n    # macro.  This avoids these false positives:\n    #  - macro that defines a base class\n    #  - multi-line macro that defines a base class\n    #  - macro that defines the whole class-head\n    #\n    # But we still issue warnings for macros that we know are safe to\n    # warn, specifically:\n    #  - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P\n    #  - TYPED_TEST\n    #  - INTERFACE_DEF\n    #  - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED:\n    #\n    # We implement a whitelist of safe macros instead of a blacklist of\n    # unsafe macros, even though the latter appears less frequently in\n    # google code and would have been easier to implement.  This is because\n    # the downside for getting the whitelist wrong means some extra\n    # semicolons, while the downside for getting the blacklist wrong\n    # would result in compile errors.\n    #\n    # In addition to macros, we also don't want to warn on\n    #  - Compound literals\n    #  - Lambdas\n    #  - alignas specifier with anonymous structs\n    #  - decltype\n    closing_brace_pos = match.group(1).rfind(')')\n    opening_parenthesis = ReverseCloseExpression(\n        clean_lines, linenum, closing_brace_pos)\n    if opening_parenthesis[2] > -1:\n      line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]]\n      macro = Search(r'\\b([A-Z_][A-Z0-9_]*)\\s*$', line_prefix)\n      func = Match(r'^(.*\\])\\s*$', line_prefix)\n      if ((macro and\n           macro.group(1) not in (\n               'TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST',\n               'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED',\n               'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or\n          (func and not Search(r'\\boperator\\s*\\[\\s*\\]', func.group(1))) or\n          Search(r'\\b(?:struct|union)\\s+alignas\\s*$', line_prefix) or\n          Search(r'\\bdecltype$', line_prefix) or\n          Search(r'\\s+=\\s*$', line_prefix)):\n        match = None\n    if (match and\n        opening_parenthesis[1] > 1 and\n        Search(r'\\]\\s*$', clean_lines.elided[opening_parenthesis[1] - 1])):\n      # Multi-line lambda-expression\n      match = None\n\n  else:\n    # Try matching cases 2-3.\n    match = Match(r'^(.*(?:else|\\)\\s*const)\\s*)\\{', line)\n    if not match:\n      # Try matching cases 4-6.  These are always matched on separate lines.\n      #\n      # Note that we can't simply concatenate the previous line to the\n      # current line and do a single match, otherwise we may output\n      # duplicate warnings for the blank line case:\n      #   if (cond) {\n      #     // blank line\n      #   }\n      prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0]\n      if prevline and Search(r'[;{}]\\s*$', prevline):\n        match = Match(r'^(\\s*)\\{', line)\n\n  # Check matching closing brace\n  if match:\n    (endline, endlinenum, endpos) = CloseExpression(\n        clean_lines, linenum, len(match.group(1)))\n    if endpos > -1 and Match(r'^\\s*;', endline[endpos:]):\n      # Current {} pair is eligible for semicolon check, and we have found\n      # the redundant semicolon, output warning here.\n      #\n      # Note: because we are scanning forward for opening braces, and\n      # outputting warnings for the matching closing brace, if there are\n      # nested blocks with trailing semicolons, we will get the error\n      # messages in reversed order.\n\n      # We need to check the line forward for NOLINT\n      raw_lines = clean_lines.raw_lines\n      ParseNolintSuppressions(filename, raw_lines[endlinenum-1], endlinenum-1,\n                              error)\n      ParseNolintSuppressions(filename, raw_lines[endlinenum], endlinenum,\n                              error)\n\n      error(filename, endlinenum, 'readability/braces', 4,\n            \"You don't need a ; after a }\")", "response": "Checks for redundant trailing semicolon."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck for empty loop body with only a single semicolon.", "response": "def CheckEmptyBlockBody(filename, clean_lines, linenum, error):\n  \"\"\"Look for empty loop/conditional body with only a single semicolon.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    error: The function to call with any errors found.\n  \"\"\"\n\n  # Search for loop keywords at the beginning of the line.  Because only\n  # whitespaces are allowed before the keywords, this will also ignore most\n  # do-while-loops, since those lines should start with closing brace.\n  #\n  # We also check \"if\" blocks here, since an empty conditional block\n  # is likely an error.\n  line = clean_lines.elided[linenum]\n  matched = Match(r'\\s*(for|while|if)\\s*\\(', line)\n  if matched:\n    # Find the end of the conditional expression.\n    (end_line, end_linenum, end_pos) = CloseExpression(\n        clean_lines, linenum, line.find('('))\n\n    # Output warning if what follows the condition expression is a semicolon.\n    # No warning for all other cases, including whitespace or newline, since we\n    # have a separate check for semicolons preceded by whitespace.\n    if end_pos >= 0 and Match(r';', end_line[end_pos:]):\n      if matched.group(1) == 'if':\n        error(filename, end_linenum, 'whitespace/empty_conditional_body', 5,\n              'Empty conditional bodies should use {}')\n      else:\n        error(filename, end_linenum, 'whitespace/empty_loop_body', 5,\n              'Empty loop bodies should use {} or continue')\n\n    # Check for if statements that have completely empty bodies (no comments)\n    # and no else clauses.\n    if end_pos >= 0 and matched.group(1) == 'if':\n      # Find the position of the opening { for the if statement.\n      # Return without logging an error if it has no brackets.\n      opening_linenum = end_linenum\n      opening_line_fragment = end_line[end_pos:]\n      # Loop until EOF or find anything that's not whitespace or opening {.\n      while not Search(r'^\\s*\\{', opening_line_fragment):\n        if Search(r'^(?!\\s*$)', opening_line_fragment):\n          # Conditional has no brackets.\n          return\n        opening_linenum += 1\n        if opening_linenum == len(clean_lines.elided):\n          # Couldn't find conditional's opening { or any code before EOF.\n          return\n        opening_line_fragment = clean_lines.elided[opening_linenum]\n      # Set opening_line (opening_line_fragment may not be entire opening line).\n      opening_line = clean_lines.elided[opening_linenum]\n\n      # Find the position of the closing }.\n      opening_pos = opening_line_fragment.find('{')\n      if opening_linenum == end_linenum:\n        # We need to make opening_pos relative to the start of the entire line.\n        opening_pos += end_pos\n      (closing_line, closing_linenum, closing_pos) = CloseExpression(\n          clean_lines, opening_linenum, opening_pos)\n      if closing_pos < 0:\n        return\n\n      # Now construct the body of the conditional. This consists of the portion\n      # of the opening line after the {, all lines until the closing line,\n      # and the portion of the closing line before the }.\n      if (clean_lines.raw_lines[opening_linenum] !=\n          CleanseComments(clean_lines.raw_lines[opening_linenum])):\n        # Opening line ends with a comment, so conditional isn't empty.\n        return\n      if closing_linenum > opening_linenum:\n        # Opening line after the {. Ignore comments here since we checked above.\n        bodylist = list(opening_line[opening_pos+1:])\n        # All lines until closing line, excluding closing line, with comments.\n        bodylist.extend(clean_lines.raw_lines[opening_linenum+1:closing_linenum])\n        # Closing line before the }. Won't (and can't) have comments.\n        bodylist.append(clean_lines.elided[closing_linenum][:closing_pos-1])\n        body = '\\n'.join(bodylist)\n      else:\n        # If statement has brackets and fits on a single line.\n        body = opening_line[opening_pos+1:closing_pos-1]\n\n      # Check if the body is empty\n      if not _EMPTY_CONDITIONAL_BODY_PATTERN.search(body):\n        return\n      # The body is empty. Now make sure there's not an else clause.\n      current_linenum = closing_linenum\n      current_line_fragment = closing_line[closing_pos:]\n      # Loop until EOF or find anything that's not whitespace or else clause.\n      while Search(r'^\\s*$|^(?=\\s*else)', current_line_fragment):\n        if Search(r'^(?=\\s*else)', current_line_fragment):\n          # Found an else clause, so don't log an error.\n          return\n        current_linenum += 1\n        if current_linenum == len(clean_lines.elided):\n          break\n        current_line_fragment = clean_lines.elided[current_linenum]\n\n      # The body is empty and there's no else clause until EOF or other code.\n      error(filename, end_linenum, 'whitespace/empty_if_body', 4,\n            ('If statement had no body and no else clause'))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef FindCheckMacro(line):\n  for macro in _CHECK_MACROS:\n    i = line.find(macro)\n    if i >= 0:\n      # Find opening parenthesis.  Do a regular expression match here\n      # to make sure that we are matching the expected CHECK macro, as\n      # opposed to some other macro that happens to contain the CHECK\n      # substring.\n      matched = Match(r'^(.*\\b' + macro + r'\\s*)\\(', line)\n      if not matched:\n        continue\n      return (macro, len(matched.group(1)))\n  return (None, -1)", "response": "Searches for a replaceable CHECK - like macro."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CheckCheck(filename, clean_lines, linenum, error):\n\n  # Decide the set of replacement macros that should be suggested\n  lines = clean_lines.elided\n  (check_macro, start_pos) = FindCheckMacro(lines[linenum])\n  if not check_macro:\n    return\n\n  # Find end of the boolean expression by matching parentheses\n  (last_line, end_line, end_pos) = CloseExpression(\n      clean_lines, linenum, start_pos)\n  if end_pos < 0:\n    return\n\n  # If the check macro is followed by something other than a\n  # semicolon, assume users will log their own custom error messages\n  # and don't suggest any replacements.\n  if not Match(r'\\s*;', last_line[end_pos:]):\n    return\n\n  if linenum == end_line:\n    expression = lines[linenum][start_pos + 1:end_pos - 1]\n  else:\n    expression = lines[linenum][start_pos + 1:]\n    for i in xrange(linenum + 1, end_line):\n      expression += lines[i]\n    expression += last_line[0:end_pos - 1]\n\n  # Parse expression so that we can take parentheses into account.\n  # This avoids false positives for inputs like \"CHECK((a < 4) == b)\",\n  # which is not replaceable by CHECK_LE.\n  lhs = ''\n  rhs = ''\n  operator = None\n  while expression:\n    matched = Match(r'^\\s*(<<|<<=|>>|>>=|->\\*|->|&&|\\|\\||'\n                    r'==|!=|>=|>|<=|<|\\()(.*)$', expression)\n    if matched:\n      token = matched.group(1)\n      if token == '(':\n        # Parenthesized operand\n        expression = matched.group(2)\n        (end, _) = FindEndOfExpressionInLine(expression, 0, ['('])\n        if end < 0:\n          return  # Unmatched parenthesis\n        lhs += '(' + expression[0:end]\n        expression = expression[end:]\n      elif token in ('&&', '||'):\n        # Logical and/or operators.  This means the expression\n        # contains more than one term, for example:\n        #   CHECK(42 < a && a < b);\n        #\n        # These are not replaceable with CHECK_LE, so bail out early.\n        return\n      elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'):\n        # Non-relational operator\n        lhs += token\n        expression = matched.group(2)\n      else:\n        # Relational operator\n        operator = token\n        rhs = matched.group(2)\n        break\n    else:\n      # Unparenthesized operand.  Instead of appending to lhs one character\n      # at a time, we do another regular expression match to consume several\n      # characters at once if possible.  Trivial benchmark shows that this\n      # is more efficient when the operands are longer than a single\n      # character, which is generally the case.\n      matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression)\n      if not matched:\n        matched = Match(r'^(\\s*\\S)(.*)$', expression)\n        if not matched:\n          break\n      lhs += matched.group(1)\n      expression = matched.group(2)\n\n  # Only apply checks if we got all parts of the boolean expression\n  if not (lhs and operator and rhs):\n    return\n\n  # Check that rhs do not contain logical operators.  We already know\n  # that lhs is fine since the loop above parses out && and ||.\n  if rhs.find('&&') > -1 or rhs.find('||') > -1:\n    return\n\n  # At least one of the operands must be a constant literal.  This is\n  # to avoid suggesting replacements for unprintable things like\n  # CHECK(variable != iterator)\n  #\n  # The following pattern matches decimal, hex integers, strings, and\n  # characters (in that order).\n  lhs = lhs.strip()\n  rhs = rhs.strip()\n  match_constant = r'^([-+]?(\\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|\".*\"|\\'.*\\')$'\n  if Match(match_constant, lhs) or Match(match_constant, rhs):\n    # Note: since we know both lhs and rhs, we can provide a more\n    # descriptive error message like:\n    #   Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42)\n    # Instead of:\n    #   Consider using CHECK_EQ instead of CHECK(a == b)\n    #\n    # We are still keeping the less descriptive message because if lhs\n    # or rhs gets long, the error message might become unreadable.\n    error(filename, linenum, 'readability/check', 2,\n          'Consider using %s instead of %s(a %s b)' % (\n              _CHECK_REPLACEMENT[check_macro][operator],\n              check_macro, operator))", "response": "Checks the use of CHECK and EXPECT macros."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef CheckAltTokens(filename, clean_lines, linenum, error):\n  line = clean_lines.elided[linenum]\n\n  # Avoid preprocessor lines\n  if Match(r'^\\s*#', line):\n    return\n\n  # Last ditch effort to avoid multi-line comments.  This will not help\n  # if the comment started before the current line or ended after the\n  # current line, but it catches most of the false positives.  At least,\n  # it provides a way to workaround this warning for people who use\n  # multi-line comments in preprocessor macros.\n  #\n  # TODO(unknown): remove this once cpplint has better support for\n  # multi-line comments.\n  if line.find('/*') >= 0 or line.find('*/') >= 0:\n    return\n\n  for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line):\n    error(filename, linenum, 'readability/alt_tokens', 2,\n          'Use operator %s instead of %s' % (\n              _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1)))", "response": "Checks alternative keywords being used in boolean expressions."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef GetLineWidth(line):\n  if isinstance(line, unicode):\n    width = 0\n    for uc in unicodedata.normalize('NFC', line):\n      if unicodedata.east_asian_width(uc) in ('W', 'F'):\n        width += 2\n      elif not unicodedata.combining(uc):\n        width += 1\n    return width\n  else:\n    return len(line)", "response": "Determines the width of a Unicode line in column positions."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndrop common suffixes like _test. cc or - inl. h from filename. Returns the filename with the common suffix removed.", "response": "def _DropCommonSuffixes(filename):\n  \"\"\"Drops common suffixes like _test.cc or -inl.h from filename.\n\n  For example:\n    >>> _DropCommonSuffixes('foo/foo-inl.h')\n    'foo/foo'\n    >>> _DropCommonSuffixes('foo/bar/foo.cc')\n    'foo/bar/foo'\n    >>> _DropCommonSuffixes('foo/foo_internal.h')\n    'foo/foo'\n    >>> _DropCommonSuffixes('foo/foo_unusualinternal.h')\n    'foo/foo_unusualinternal'\n\n  Args:\n    filename: The input filename.\n\n  Returns:\n    The filename with the common suffix removed.\n  \"\"\"\n  for suffix in itertools.chain(\n      ('%s.%s' % (test_suffix.lstrip('_'), ext)\n       for test_suffix, ext in itertools.product(_test_suffixes, GetNonHeaderExtensions())),\n      ('%s.%s' % (suffix, ext)\n       for suffix, ext in itertools.product(['inl', 'imp', 'internal'], GetHeaderExtensions()))):\n    if (filename.endswith(suffix) and len(filename) > len(suffix) and\n        filename[-len(suffix) - 1] in ('-', '_')):\n      return filename[:-len(suffix) - 1]\n  return os.path.splitext(filename)[0]"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nfigure out what kind of header include is.", "response": "def _ClassifyInclude(fileinfo, include, is_system):\n  \"\"\"Figures out what kind of header 'include' is.\n\n  Args:\n    fileinfo: The current file cpplint is running over. A FileInfo instance.\n    include: The path to a #included file.\n    is_system: True if the #include used <> rather than \"\".\n\n  Returns:\n    One of the _XXX_HEADER constants.\n\n  For example:\n    >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'stdio.h', True)\n    _C_SYS_HEADER\n    >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'string', True)\n    _CPP_SYS_HEADER\n    >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/foo.h', False)\n    _LIKELY_MY_HEADER\n    >>> _ClassifyInclude(FileInfo('foo/foo_unknown_extension.cc'),\n    ...                  'bar/foo_other_ext.h', False)\n    _POSSIBLE_MY_HEADER\n    >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False)\n    _OTHER_HEADER\n  \"\"\"\n  # This is a list of all standard c++ header files, except\n  # those already checked for above.\n  is_cpp_h = include in _CPP_HEADERS\n\n  # Headers with C++ extensions shouldn't be considered C system headers\n  if is_system and os.path.splitext(include)[1] in ['.hpp', '.hxx', '.h++']:\n      is_system = False\n\n  if is_system:\n    if is_cpp_h:\n      return _CPP_SYS_HEADER\n    else:\n      return _C_SYS_HEADER\n\n  # If the target file and the include we're checking share a\n  # basename when we drop common extensions, and the include\n  # lives in . , then it's likely to be owned by the target file.\n  target_dir, target_base = (\n      os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName())))\n  include_dir, include_base = os.path.split(_DropCommonSuffixes(include))\n  target_dir_pub = os.path.normpath(target_dir + '/../public')\n  target_dir_pub = target_dir_pub.replace('\\\\', '/')\n  if target_base == include_base and (\n      include_dir == target_dir or\n      include_dir == target_dir_pub):\n    return _LIKELY_MY_HEADER\n\n  # If the target and include share some initial basename\n  # component, it's possible the target is implementing the\n  # include, so it's allowed to be first, but we'll never\n  # complain if it's not there.\n  target_first_component = _RE_FIRST_COMPONENT.match(target_base)\n  include_first_component = _RE_FIRST_COMPONENT.match(include_base)\n  if (target_first_component and include_first_component and\n      target_first_component.group(0) ==\n      include_first_component.group(0)):\n    return _POSSIBLE_MY_HEADER\n\n  return _OTHER_HEADER"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks if the line is applicable to include lines.", "response": "def CheckIncludeLine(filename, clean_lines, linenum, include_state, error):\n  \"\"\"Check rules that are applicable to #include lines.\n\n  Strings on #include lines are NOT removed from elided line, to make\n  certain tasks easier. However, to prevent false positives, checks\n  applicable to #include lines in CheckLanguage must be put here.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    include_state: An _IncludeState instance in which the headers are inserted.\n    error: The function to call with any errors found.\n  \"\"\"\n  fileinfo = FileInfo(filename)\n  line = clean_lines.lines[linenum]\n\n  # \"include\" should use the new style \"foo/bar.h\" instead of just \"bar.h\"\n  # Only do this check if the included header follows google naming\n  # conventions.  If not, assume that it's a 3rd party API that\n  # requires special include conventions.\n  #\n  # We also make an exception for Lua headers, which follow google\n  # naming convention but not the include convention.\n  match = Match(r'#include\\s*\"([^/]+\\.h)\"', line)\n  if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)):\n    error(filename, linenum, 'build/include_subdir', 4,\n          'Include the directory when naming .h files')\n\n  # we shouldn't include a file more than once. actually, there are a\n  # handful of instances where doing so is okay, but in general it's\n  # not.\n  match = _RE_PATTERN_INCLUDE.search(line)\n  if match:\n    include = match.group(2)\n    is_system = (match.group(1) == '<')\n    duplicate_line = include_state.FindHeader(include)\n    if duplicate_line >= 0:\n      error(filename, linenum, 'build/include', 4,\n            '\"%s\" already included at %s:%s' %\n            (include, filename, duplicate_line))\n      return\n\n    for extension in GetNonHeaderExtensions():\n      if (include.endswith('.' + extension) and\n          os.path.dirname(fileinfo.RepositoryName()) != os.path.dirname(include)):\n        error(filename, linenum, 'build/include', 4,\n              'Do not include .' + extension + ' files from other packages')\n        return\n\n    if not _THIRD_PARTY_HEADERS_PATTERN.match(include):\n      include_state.include_list[-1].append((include, linenum))\n\n      # We want to ensure that headers appear in the right order:\n      # 1) for foo.cc, foo.h  (preferred location)\n      # 2) c system files\n      # 3) cpp system files\n      # 4) for foo.cc, foo.h  (deprecated location)\n      # 5) other google headers\n      #\n      # We classify each include statement as one of those 5 types\n      # using a number of techniques. The include_state object keeps\n      # track of the highest type seen, and complains if we see a\n      # lower type after that.\n      error_message = include_state.CheckNextIncludeOrder(\n          _ClassifyInclude(fileinfo, include, is_system))\n      if error_message:\n        error(filename, linenum, 'build/include_order', 4,\n              '%s. Should be: %s.h, c system, c++ system, other.' %\n              (error_message, fileinfo.BaseName()))\n      canonical_include = include_state.CanonicalizeAlphabeticalOrder(include)\n      if not include_state.IsInAlphabeticalOrder(\n          clean_lines, linenum, canonical_include):\n        error(filename, linenum, 'build/include_alpha', 4,\n              'Include \"%s\" not in alphabetical order' % include)\n      include_state.SetLastHeader(canonical_include)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef CheckLanguage(filename, clean_lines, linenum, file_extension,\n                  include_state, nesting_state, error):\n  \"\"\"Checks rules from the 'C++ language rules' section of cppguide.html.\n\n  Some of these rules are hard to test (function overloading, using\n  uint32 inappropriately), but we do the best we can.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    file_extension: The extension (without the dot) of the filename.\n    include_state: An _IncludeState instance in which the headers are inserted.\n    nesting_state: A NestingState instance which maintains information about\n                   the current stack of nested blocks being parsed.\n    error: The function to call with any errors found.\n  \"\"\"\n  # If the line is empty or consists of entirely a comment, no need to\n  # check it.\n  line = clean_lines.elided[linenum]\n  if not line:\n    return\n\n  match = _RE_PATTERN_INCLUDE.search(line)\n  if match:\n    CheckIncludeLine(filename, clean_lines, linenum, include_state, error)\n    return\n\n  # Reset include state across preprocessor directives.  This is meant\n  # to silence warnings for conditional includes.\n  match = Match(r'^\\s*#\\s*(if|ifdef|ifndef|elif|else|endif)\\b', line)\n  if match:\n    include_state.ResetSection(match.group(1))\n\n\n  # Perform other checks now that we are sure that this is not an include line\n  CheckCasts(filename, clean_lines, linenum, error)\n  CheckGlobalStatic(filename, clean_lines, linenum, error)\n  CheckPrintf(filename, clean_lines, linenum, error)\n\n  if file_extension in GetHeaderExtensions():\n    # TODO(unknown): check that 1-arg constructors are explicit.\n    #                How to tell it's a constructor?\n    #                (handled in CheckForNonStandardConstructs for now)\n    # TODO(unknown): check that classes declare or disable copy/assign\n    #                (level 1 error)\n    pass\n\n  # Check if people are using the verboten C basic types.  The only exception\n  # we regularly allow is \"unsigned short port\" for port.\n  if Search(r'\\bshort port\\b', line):\n    if not Search(r'\\bunsigned short port\\b', line):\n      error(filename, linenum, 'runtime/int', 4,\n            'Use \"unsigned short\" for ports, not \"short\"')\n  else:\n    match = Search(r'\\b(short|long(?! +double)|long long)\\b', line)\n    if match:\n      error(filename, linenum, 'runtime/int', 4,\n            'Use int16/int64/etc, rather than the C type %s' % match.group(1))\n\n  # Check if some verboten operator overloading is going on\n  # TODO(unknown): catch out-of-line unary operator&:\n  #   class X {};\n  #   int operator&(const X& x) { return 42; }  // unary operator&\n  # The trick is it's hard to tell apart from binary operator&:\n  #   class Y { int operator&(const Y& x) { return 23; } }; // binary operator&\n  if Search(r'\\boperator\\s*&\\s*\\(\\s*\\)', line):\n    error(filename, linenum, 'runtime/operator', 4,\n          'Unary operator& is dangerous.  Do not use it.')\n\n  # Check for suspicious usage of \"if\" like\n  # } if (a == b) {\n  if Search(r'\\}\\s*if\\s*\\(', line):\n    error(filename, linenum, 'readability/braces', 4,\n          'Did you mean \"else if\"? If not, start a new line for \"if\".')\n\n  # Check for potential format string bugs like printf(foo).\n  # We constrain the pattern not to pick things like DocidForPrintf(foo).\n  # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str())\n  # TODO(unknown): Catch the following case. Need to change the calling\n  # convention of the whole function to process multiple line to handle it.\n  #   printf(\n  #       boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line);\n  printf_args = _GetTextInside(line, r'(?i)\\b(string)?printf\\s*\\(')\n  if printf_args:\n    match = Match(r'([\\w.\\->()]+)$', printf_args)\n    if match and match.group(1) != '__VA_ARGS__':\n      function_name = re.search(r'\\b((?:string)?printf)\\s*\\(',\n                                line, re.I).group(1)\n      error(filename, linenum, 'runtime/printf', 4,\n            'Potential format string bug. Do %s(\"%%s\", %s) instead.'\n            % (function_name, match.group(1)))\n\n  # Check for potential memset bugs like memset(buf, sizeof(buf), 0).\n  match = Search(r'memset\\s*\\(([^,]*),\\s*([^,]*),\\s*0\\s*\\)', line)\n  if match and not Match(r\"^''|-?[0-9]+|0x[0-9A-Fa-f]$\", match.group(2)):\n    error(filename, linenum, 'runtime/memset', 4,\n          'Did you mean \"memset(%s, 0, %s)\"?'\n          % (match.group(1), match.group(2)))\n\n  if Search(r'\\busing namespace\\b', line):\n    if Search(r'\\bliterals\\b', line):\n      error(filename, linenum, 'build/namespaces_literals', 5,\n            'Do not use namespace using-directives.  '\n            'Use using-declarations instead.')\n    else:\n      error(filename, linenum, 'build/namespaces', 5,\n            'Do not use namespace using-directives.  '\n            'Use using-declarations instead.')\n\n  # Detect variable-length arrays.\n  match = Match(r'\\s*(.+::)?(\\w+) [a-z]\\w*\\[(.+)];', line)\n  if (match and match.group(2) != 'return' and match.group(2) != 'delete' and\n      match.group(3).find(']') == -1):\n    # Split the size using space and arithmetic operators as delimiters.\n    # If any of the resulting tokens are not compile time constants then\n    # report the error.\n    tokens = re.split(r'\\s|\\+|\\-|\\*|\\/|<<|>>]', match.group(3))\n    is_const = True\n    skip_next = False\n    for tok in tokens:\n      if skip_next:\n        skip_next = False\n        continue\n\n      if Search(r'sizeof\\(.+\\)', tok): continue\n      if Search(r'arraysize\\(\\w+\\)', tok): continue\n\n      tok = tok.lstrip('(')\n      tok = tok.rstrip(')')\n      if not tok: continue\n      if Match(r'\\d+', tok): continue\n      if Match(r'0[xX][0-9a-fA-F]+', tok): continue\n      if Match(r'k[A-Z0-9]\\w*', tok): continue\n      if Match(r'(.+::)?k[A-Z0-9]\\w*', tok): continue\n      if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue\n      # A catch all for tricky sizeof cases, including 'sizeof expression',\n      # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)'\n      # requires skipping the next token because we split on ' ' and '*'.\n      if tok.startswith('sizeof'):\n        skip_next = True\n        continue\n      is_const = False\n      break\n    if not is_const:\n      error(filename, linenum, 'runtime/arrays', 1,\n            'Do not use variable-length arrays.  Use an appropriately named '\n            \"('k' followed by CamelCase) compile-time constant for the size.\")\n\n  # Check for use of unnamed namespaces in header files.  Registration\n  # macros are typically OK, so we allow use of \"namespace {\" on lines\n  # that end with backslashes.\n  if (file_extension in GetHeaderExtensions()\n      and Search(r'\\bnamespace\\s*{', line)\n      and line[-1] != '\\\\'):\n    error(filename, linenum, 'build/namespaces', 4,\n          'Do not use unnamed namespaces in header files.  See '\n          'https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces'\n          ' for more information.')", "response": "Checks the rules from the C ++ language rules section of cppguide. html."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncheck for unsafe global or static objects at the current line.", "response": "def CheckGlobalStatic(filename, clean_lines, linenum, error):\n  \"\"\"Check for unsafe global or static objects.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    error: The function to call with any errors found.\n  \"\"\"\n  line = clean_lines.elided[linenum]\n\n  # Match two lines at a time to support multiline declarations\n  if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line):\n    line += clean_lines.elided[linenum + 1].strip()\n\n  # Check for people declaring static/global STL strings at the top level.\n  # This is dangerous because the C++ language does not guarantee that\n  # globals with constructors are initialized before the first access, and\n  # also because globals can be destroyed when some threads are still running.\n  # TODO(unknown): Generalize this to also find static unique_ptr instances.\n  # TODO(unknown): File bugs for clang-tidy to find these.\n  match = Match(\n      r'((?:|static +)(?:|const +))(?::*std::)?string( +const)? +'\n      r'([a-zA-Z0-9_:]+)\\b(.*)',\n      line)\n\n  # Remove false positives:\n  # - String pointers (as opposed to values).\n  #    string *pointer\n  #    const string *pointer\n  #    string const *pointer\n  #    string *const pointer\n  #\n  # - Functions and template specializations.\n  #    string Function<Type>(...\n  #    string Class<Type>::Method(...\n  #\n  # - Operators.  These are matched separately because operator names\n  #   cross non-word boundaries, and trying to match both operators\n  #   and functions at the same time would decrease accuracy of\n  #   matching identifiers.\n  #    string Class::operator*()\n  if (match and\n      not Search(r'\\bstring\\b(\\s+const)?\\s*[\\*\\&]\\s*(const\\s+)?\\w', line) and\n      not Search(r'\\boperator\\W', line) and\n      not Match(r'\\s*(<.*>)?(::[a-zA-Z0-9_]+)*\\s*\\(([^\"]|$)', match.group(4))):\n    if Search(r'\\bconst\\b', line):\n      error(filename, linenum, 'runtime/string', 4,\n            'For a static/global string constant, use a C style string '\n            'instead: \"%schar%s %s[]\".' %\n            (match.group(1), match.group(2) or '', match.group(3)))\n    else:\n      error(filename, linenum, 'runtime/string', 4,\n            'Static/global string variables are not permitted.')\n\n  if (Search(r'\\b([A-Za-z0-9_]*_)\\(\\1\\)', line) or\n      Search(r'\\b([A-Za-z0-9_]*_)\\(CHECK_NOTNULL\\(\\1\\)\\)', line)):\n    error(filename, linenum, 'runtime/init', 4,\n          'You seem to be initializing a member variable with itself.')"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck for printf related issues.", "response": "def CheckPrintf(filename, clean_lines, linenum, error):\n  \"\"\"Check for printf related issues.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    error: The function to call with any errors found.\n  \"\"\"\n  line = clean_lines.elided[linenum]\n\n  # When snprintf is used, the second argument shouldn't be a literal.\n  match = Search(r'snprintf\\s*\\(([^,]*),\\s*([0-9]*)\\s*,', line)\n  if match and match.group(2) != '0':\n    # If 2nd arg is zero, snprintf is used to calculate size.\n    error(filename, linenum, 'runtime/printf', 3,\n          'If you can, use sizeof(%s) instead of %s as the 2nd arg '\n          'to snprintf.' % (match.group(1), match.group(2)))\n\n  # Check if some verboten C functions are being used.\n  if Search(r'\\bsprintf\\s*\\(', line):\n    error(filename, linenum, 'runtime/printf', 5,\n          'Never use sprintf. Use snprintf instead.')\n  match = Search(r'\\b(strcpy|strcat)\\s*\\(', line)\n  if match:\n    error(filename, linenum, 'runtime/printf', 4,\n          'Almost always, snprintf is better than %s' % match.group(1))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef IsDerivedFunction(clean_lines, linenum):\n  # Scan back a few lines for start of current function\n  for i in xrange(linenum, max(-1, linenum - 10), -1):\n    match = Match(r'^([^()]*\\w+)\\(', clean_lines.elided[i])\n    if match:\n      # Look for \"override\" after the matching closing parenthesis\n      line, _, closing_paren = CloseExpression(\n          clean_lines, i, len(match.group(1)))\n      return (closing_paren >= 0 and\n              Search(r'\\boverride\\b', line[closing_paren:]))\n  return False", "response": "Checks if the current line contains an inherited function."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchecking if the current line contains an out - of - line method definition.", "response": "def IsOutOfLineMethodDefinition(clean_lines, linenum):\n  \"\"\"Check if current line contains an out-of-line method definition.\n\n  Args:\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n  Returns:\n    True if current line contains an out-of-line method definition.\n  \"\"\"\n  # Scan back a few lines for start of current function\n  for i in xrange(linenum, max(-1, linenum - 10), -1):\n    if Match(r'^([^()]*\\w+)\\(', clean_lines.elided[i]):\n      return Match(r'^[^()]*\\w+::\\w+\\(', clean_lines.elided[i]) is not None\n  return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef IsInitializerList(clean_lines, linenum):\n  for i in xrange(linenum, 1, -1):\n    line = clean_lines.elided[i]\n    if i == linenum:\n      remove_function_body = Match(r'^(.*)\\{\\s*$', line)\n      if remove_function_body:\n        line = remove_function_body.group(1)\n\n    if Search(r'\\s:\\s*\\w+[({]', line):\n      # A lone colon tend to indicate the start of a constructor\n      # initializer list.  It could also be a ternary operator, which\n      # also tend to appear in constructor initializer lists as\n      # opposed to parameter lists.\n      return True\n    if Search(r'\\}\\s*,\\s*$', line):\n      # A closing brace followed by a comma is probably the end of a\n      # brace-initialized member in constructor initializer list.\n      return True\n    if Search(r'[{};]\\s*$', line):\n      # Found one of the following:\n      # - A closing brace or semicolon, probably the end of the previous\n      #   function.\n      # - An opening brace, probably the start of current class or namespace.\n      #\n      # Current line is probably not inside an initializer list since\n      # we saw one of those things without seeing the starting colon.\n      return False\n\n  # Got to the beginning of the file without seeing the start of\n  # constructor initializer list.\n  return False", "response": "Checks if current line is inside constructor initializer list."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CheckForNonConstReference(filename, clean_lines, linenum,\n                              nesting_state, error):\n  \"\"\"Check for non-const references.\n\n  Separate from CheckLanguage since it scans backwards from current\n  line, instead of scanning forward.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    nesting_state: A NestingState instance which maintains information about\n                   the current stack of nested blocks being parsed.\n    error: The function to call with any errors found.\n  \"\"\"\n  # Do nothing if there is no '&' on current line.\n  line = clean_lines.elided[linenum]\n  if '&' not in line:\n    return\n\n  # If a function is inherited, current function doesn't have much of\n  # a choice, so any non-const references should not be blamed on\n  # derived function.\n  if IsDerivedFunction(clean_lines, linenum):\n    return\n\n  # Don't warn on out-of-line method definitions, as we would warn on the\n  # in-line declaration, if it isn't marked with 'override'.\n  if IsOutOfLineMethodDefinition(clean_lines, linenum):\n    return\n\n  # Long type names may be broken across multiple lines, usually in one\n  # of these forms:\n  #   LongType\n  #       ::LongTypeContinued &identifier\n  #   LongType::\n  #       LongTypeContinued &identifier\n  #   LongType<\n  #       ...>::LongTypeContinued &identifier\n  #\n  # If we detected a type split across two lines, join the previous\n  # line to current line so that we can match const references\n  # accordingly.\n  #\n  # Note that this only scans back one line, since scanning back\n  # arbitrary number of lines would be expensive.  If you have a type\n  # that spans more than 2 lines, please use a typedef.\n  if linenum > 1:\n    previous = None\n    if Match(r'\\s*::(?:[\\w<>]|::)+\\s*&\\s*\\S', line):\n      # previous_line\\n + ::current_line\n      previous = Search(r'\\b((?:const\\s*)?(?:[\\w<>]|::)+[\\w<>])\\s*$',\n                        clean_lines.elided[linenum - 1])\n    elif Match(r'\\s*[a-zA-Z_]([\\w<>]|::)+\\s*&\\s*\\S', line):\n      # previous_line::\\n + current_line\n      previous = Search(r'\\b((?:const\\s*)?(?:[\\w<>]|::)+::)\\s*$',\n                        clean_lines.elided[linenum - 1])\n    if previous:\n      line = previous.group(1) + line.lstrip()\n    else:\n      # Check for templated parameter that is split across multiple lines\n      endpos = line.rfind('>')\n      if endpos > -1:\n        (_, startline, startpos) = ReverseCloseExpression(\n            clean_lines, linenum, endpos)\n        if startpos > -1 and startline < linenum:\n          # Found the matching < on an earlier line, collect all\n          # pieces up to current line.\n          line = ''\n          for i in xrange(startline, linenum + 1):\n            line += clean_lines.elided[i].strip()\n\n  # Check for non-const references in function parameters.  A single '&' may\n  # found in the following places:\n  #   inside expression: binary & for bitwise AND\n  #   inside expression: unary & for taking the address of something\n  #   inside declarators: reference parameter\n  # We will exclude the first two cases by checking that we are not inside a\n  # function body, including one that was just introduced by a trailing '{'.\n  # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare].\n  if (nesting_state.previous_stack_top and\n      not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or\n           isinstance(nesting_state.previous_stack_top, _NamespaceInfo))):\n    # Not at toplevel, not within a class, and not within a namespace\n    return\n\n  # Avoid initializer lists.  We only need to scan back from the\n  # current line for something that starts with ':'.\n  #\n  # We don't need to check the current line, since the '&' would\n  # appear inside the second set of parentheses on the current line as\n  # opposed to the first set.\n  if linenum > 0:\n    for i in xrange(linenum - 1, max(0, linenum - 10), -1):\n      previous_line = clean_lines.elided[i]\n      if not Search(r'[),]\\s*$', previous_line):\n        break\n      if Match(r'^\\s*:\\s+\\S', previous_line):\n        return\n\n  # Avoid preprocessors\n  if Search(r'\\\\\\s*$', line):\n    return\n\n  # Avoid constructor initializer lists\n  if IsInitializerList(clean_lines, linenum):\n    return\n\n  # We allow non-const references in a few standard places, like functions\n  # called \"swap()\" or iostream operators like \"<<\" or \">>\".  Do not check\n  # those function parameters.\n  #\n  # We also accept & in static_assert, which looks like a function but\n  # it's actually a declaration expression.\n  whitelisted_functions = (r'(?:[sS]wap(?:<\\w:+>)?|'\n                           r'operator\\s*[<>][<>]|'\n                           r'static_assert|COMPILE_ASSERT'\n                           r')\\s*\\(')\n  if Search(whitelisted_functions, line):\n    return\n  elif not Search(r'\\S+\\([^)]*$', line):\n    # Don't see a whitelisted function on this line.  Actually we\n    # didn't see any function name on this line, so this is likely a\n    # multi-line parameter list.  Try a bit harder to catch this case.\n    for i in xrange(2):\n      if (linenum > i and\n          Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])):\n        return\n\n  decls = ReplaceAll(r'{[^}]*}', ' ', line)  # exclude function body\n  for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls):\n    if (not Match(_RE_PATTERN_CONST_REF_PARAM, parameter) and\n        not Match(_RE_PATTERN_REF_STREAM_PARAM, parameter)):\n      error(filename, linenum, 'runtime/references', 2,\n            'Is this a non-const reference? '\n            'If so, make const or use a pointer: ' +\n            ReplaceAll(' *<', '<', parameter))", "response": "Checks if a given line is a non - constant reference."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef CheckCasts(filename, clean_lines, linenum, error):\n  line = clean_lines.elided[linenum]\n\n  # Check to see if they're using an conversion function cast.\n  # I just try to capture the most common basic types, though there are more.\n  # Parameterless conversion functions, such as bool(), are allowed as they are\n  # probably a member operator declaration or default constructor.\n  match = Search(\n      r'(\\bnew\\s+(?:const\\s+)?|\\S<\\s*(?:const\\s+)?)?\\b'\n      r'(int|float|double|bool|char|int32|uint32|int64|uint64)'\n      r'(\\([^)].*)', line)\n  expecting_function = ExpectingFunctionArgs(clean_lines, linenum)\n  if match and not expecting_function:\n    matched_type = match.group(2)\n\n    # matched_new_or_template is used to silence two false positives:\n    # - New operators\n    # - Template arguments with function types\n    #\n    # For template arguments, we match on types immediately following\n    # an opening bracket without any spaces.  This is a fast way to\n    # silence the common case where the function type is the first\n    # template argument.  False negative with less-than comparison is\n    # avoided because those operators are usually followed by a space.\n    #\n    #   function<double(double)>   // bracket + no space = false positive\n    #   value < double(42)         // bracket + space = true positive\n    matched_new_or_template = match.group(1)\n\n    # Avoid arrays by looking for brackets that come after the closing\n    # parenthesis.\n    if Match(r'\\([^()]+\\)\\s*\\[', match.group(3)):\n      return\n\n    # Other things to ignore:\n    # - Function pointers\n    # - Casts to pointer types\n    # - Placement new\n    # - Alias declarations\n    matched_funcptr = match.group(3)\n    if (matched_new_or_template is None and\n        not (matched_funcptr and\n             (Match(r'\\((?:[^() ]+::\\s*\\*\\s*)?[^() ]+\\)\\s*\\(',\n                    matched_funcptr) or\n              matched_funcptr.startswith('(*)'))) and\n        not Match(r'\\s*using\\s+\\S+\\s*=\\s*' + matched_type, line) and\n        not Search(r'new\\(\\S+\\)\\s*' + matched_type, line)):\n      error(filename, linenum, 'readability/casting', 4,\n            'Using deprecated casting style.  '\n            'Use static_cast<%s>(...) instead' %\n            matched_type)\n\n  if not expecting_function:\n    CheckCStyleCast(filename, clean_lines, linenum, 'static_cast',\n                    r'\\((int|float|double|bool|char|u?int(16|32|64))\\)', error)\n\n  # This doesn't catch all cases. Consider (const char * const)\"hello\".\n  #\n  # (char *) \"foo\" should always be a const_cast (reinterpret_cast won't\n  # compile).\n  if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast',\n                     r'\\((char\\s?\\*+\\s?)\\)\\s*\"', error):\n    pass\n  else:\n    # Check pointer casts for other than string constants\n    CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast',\n                    r'\\((\\w+\\s?\\*+\\s?)\\)', error)\n\n  # In addition, we look for people taking the address of a cast.  This\n  # is dangerous -- casts can assign to temporaries, so the pointer doesn't\n  # point where you think.\n  #\n  # Some non-identifier character is required before the '&' for the\n  # expression to be recognized as a cast.  These are casts:\n  #   expression = &static_cast<int*>(temporary());\n  #   function(&(int*)(temporary()));\n  #\n  # This is not a cast:\n  #   reference_type&(int* function_param);\n  match = Search(\n      r'(?:[^\\w]&\\(([^)*][^)]*)\\)[\\w(])|'\n      r'(?:[^\\w]&(static|dynamic|down|reinterpret)_cast\\b)', line)\n  if match:\n    # Try a better error message when the & is bound to something\n    # dereferenced by the casted pointer, as opposed to the casted\n    # pointer itself.\n    parenthesis_error = False\n    match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\\b)<', line)\n    if match:\n      _, y1, x1 = CloseExpression(clean_lines, linenum, len(match.group(1)))\n      if x1 >= 0 and clean_lines.elided[y1][x1] == '(':\n        _, y2, x2 = CloseExpression(clean_lines, y1, x1)\n        if x2 >= 0:\n          extended_line = clean_lines.elided[y2][x2:]\n          if y2 < clean_lines.NumLines() - 1:\n            extended_line += clean_lines.elided[y2 + 1]\n          if Match(r'\\s*(?:->|\\[)', extended_line):\n            parenthesis_error = True\n\n    if parenthesis_error:\n      error(filename, linenum, 'readability/casting', 4,\n            ('Are you taking an address of something dereferenced '\n             'from a cast?  Wrapping the dereferenced expression in '\n             'parentheses will make the binding more obvious'))\n    else:\n      error(filename, linenum, 'runtime/casting', 4,\n            ('Are you taking an address of a cast?  '\n             'This is dangerous: could be a temp var.  '\n             'Take the address before doing the cast, rather than after'))", "response": "Checks if the current file contains a valid conversion function cast."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks if a C - style cast is found.", "response": "def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error):\n  \"\"\"Checks for a C-style cast by looking for the pattern.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    cast_type: The string for the C++ cast to recommend.  This is either\n      reinterpret_cast, static_cast, or const_cast, depending.\n    pattern: The regular expression used to find C-style casts.\n    error: The function to call with any errors found.\n\n  Returns:\n    True if an error was emitted.\n    False otherwise.\n  \"\"\"\n  line = clean_lines.elided[linenum]\n  match = Search(pattern, line)\n  if not match:\n    return False\n\n  # Exclude lines with keywords that tend to look like casts\n  context = line[0:match.start(1) - 1]\n  if Match(r'.*\\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\\s*$', context):\n    return False\n\n  # Try expanding current context to see if we one level of\n  # parentheses inside a macro.\n  if linenum > 0:\n    for i in xrange(linenum - 1, max(0, linenum - 5), -1):\n      context = clean_lines.elided[i] + context\n  if Match(r'.*\\b[_A-Z][_A-Z0-9]*\\s*\\((?:\\([^()]*\\)|[^()])*$', context):\n    return False\n\n  # operator++(int) and operator--(int)\n  if context.endswith(' operator++') or context.endswith(' operator--'):\n    return False\n\n  # A single unnamed argument for a function tends to look like old style cast.\n  # If we see those, don't issue warnings for deprecated casts.\n  remainder = line[match.end(0):]\n  if Match(r'^\\s*(?:;|const\\b|throw\\b|final\\b|override\\b|[=>{),]|->)',\n           remainder):\n    return False\n\n  # At this point, all that should be left is actual casts.\n  error(filename, linenum, 'readability/casting', 4,\n        'Using C-style cast.  Use %s<%s>(...) instead' %\n        (cast_type, match.group(1)))\n\n  return True"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef ExpectingFunctionArgs(clean_lines, linenum):\n  line = clean_lines.elided[linenum]\n  return (Match(r'^\\s*MOCK_(CONST_)?METHOD\\d+(_T)?\\(', line) or\n          (linenum >= 2 and\n           (Match(r'^\\s*MOCK_(?:CONST_)?METHOD\\d+(?:_T)?\\((?:\\S+,)?\\s*$',\n                  clean_lines.elided[linenum - 1]) or\n            Match(r'^\\s*MOCK_(?:CONST_)?METHOD\\d+(?:_T)?\\(\\s*$',\n                  clean_lines.elided[linenum - 2]) or\n            Search(r'\\bstd::m?function\\s*\\<\\s*$',\n                   clean_lines.elided[linenum - 1]))))", "response": "Checks whether the line at linenum is inside a function type argument list."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nchecks if two files belong to the same module.", "response": "def FilesBelongToSameModule(filename_cc, filename_h):\n  \"\"\"Check if these two filenames belong to the same module.\n\n  The concept of a 'module' here is a as follows:\n  foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the\n  same 'module' if they are in the same directory.\n  some/path/public/xyzzy and some/path/internal/xyzzy are also considered\n  to belong to the same module here.\n\n  If the filename_cc contains a longer path than the filename_h, for example,\n  '/absolute/path/to/base/sysinfo.cc', and this file would include\n  'base/sysinfo.h', this function also produces the prefix needed to open the\n  header. This is used by the caller of this function to more robustly open the\n  header file. We don't have access to the real include paths in this context,\n  so we need this guesswork here.\n\n  Known bugs: tools/base/bar.cc and base/bar.h belong to the same module\n  according to this implementation. Because of this, this function gives\n  some false positives. This should be sufficiently rare in practice.\n\n  Args:\n    filename_cc: is the path for the source (e.g. .cc) file\n    filename_h: is the path for the header path\n\n  Returns:\n    Tuple with a bool and a string:\n    bool: True if filename_cc and filename_h belong to the same module.\n    string: the additional prefix needed to open the header file.\n  \"\"\"\n  fileinfo_cc = FileInfo(filename_cc)\n  if not fileinfo_cc.Extension().lstrip('.') in GetNonHeaderExtensions():\n    return (False, '')\n\n  fileinfo_h = FileInfo(filename_h)\n  if not fileinfo_h.Extension().lstrip('.') in GetHeaderExtensions():\n    return (False, '')\n\n  filename_cc = filename_cc[:-(len(fileinfo_cc.Extension()))]\n  matched_test_suffix = Search(_TEST_FILE_SUFFIX, fileinfo_cc.BaseName())\n  if matched_test_suffix:\n    filename_cc = filename_cc[:-len(matched_test_suffix.group(1))]\n\n  filename_cc = filename_cc.replace('/public/', '/')\n  filename_cc = filename_cc.replace('/internal/', '/')\n\n  filename_h = filename_h[:-(len(fileinfo_h.Extension()))]\n  if filename_h.endswith('-inl'):\n    filename_h = filename_h[:-len('-inl')]\n  filename_h = filename_h.replace('/public/', '/')\n  filename_h = filename_h.replace('/internal/', '/')\n\n  files_belong_to_same_module = filename_cc.endswith(filename_h)\n  common_path = ''\n  if files_belong_to_same_module:\n    common_path = filename_cc[:-len(filename_h)]\n  return files_belong_to_same_module, common_path"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef UpdateIncludeState(filename, include_dict, io=codecs):\n  headerfile = None\n  try:\n    headerfile = io.open(filename, 'r', 'utf8', 'replace')\n  except IOError:\n    return False\n  linenum = 0\n  for line in headerfile:\n    linenum += 1\n    clean_line = CleanseComments(line)\n    match = _RE_PATTERN_INCLUDE.search(clean_line)\n    if match:\n      include = match.group(2)\n      include_dict.setdefault(include, linenum)\n  return True", "response": "Updates the include_dict with the lines found in the file."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CheckMakePairUsesDeduction(filename, clean_lines, linenum, error):\n  line = clean_lines.elided[linenum]\n  match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line)\n  if match:\n    error(filename, linenum, 'build/explicit_make_pair',\n          4,  # 4 = high confidence\n          'For C++11-compatibility, omit template arguments from make_pair'\n          ' OR use pair directly OR if appropriate, construct a pair directly')", "response": "Checks that make_pair s template arguments are deduced."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CheckRedundantVirtual(filename, clean_lines, linenum, error):\n  # Look for \"virtual\" on current line.\n  line = clean_lines.elided[linenum]\n  virtual = Match(r'^(.*)(\\bvirtual\\b)(.*)$', line)\n  if not virtual: return\n\n  # Ignore \"virtual\" keywords that are near access-specifiers.  These\n  # are only used in class base-specifier and do not apply to member\n  # functions.\n  if (Search(r'\\b(public|protected|private)\\s+$', virtual.group(1)) or\n      Match(r'^\\s+(public|protected|private)\\b', virtual.group(3))):\n    return\n\n  # Ignore the \"virtual\" keyword from virtual base classes.  Usually\n  # there is a column on the same line in these cases (virtual base\n  # classes are rare in google3 because multiple inheritance is rare).\n  if Match(r'^.*[^:]:[^:].*$', line): return\n\n  # Look for the next opening parenthesis.  This is the start of the\n  # parameter list (possibly on the next line shortly after virtual).\n  # TODO(unknown): doesn't work if there are virtual functions with\n  # decltype() or other things that use parentheses, but csearch suggests\n  # that this is rare.\n  end_col = -1\n  end_line = -1\n  start_col = len(virtual.group(2))\n  for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())):\n    line = clean_lines.elided[start_line][start_col:]\n    parameter_list = Match(r'^([^(]*)\\(', line)\n    if parameter_list:\n      # Match parentheses to find the end of the parameter list\n      (_, end_line, end_col) = CloseExpression(\n          clean_lines, start_line, start_col + len(parameter_list.group(1)))\n      break\n    start_col = 0\n\n  if end_col < 0:\n    return  # Couldn't find end of parameter list, give up\n\n  # Look for \"override\" or \"final\" after the parameter list\n  # (possibly on the next few lines).\n  for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())):\n    line = clean_lines.elided[i][end_col:]\n    match = Search(r'\\b(override|final)\\b', line)\n    if match:\n      error(filename, linenum, 'readability/inheritance', 4,\n            ('\"virtual\" is redundant since function is '\n             'already declared as \"%s\"' % match.group(1)))\n\n    # Set end_col to check whole lines after we are done with the\n    # first line.\n    end_col = 0\n    if Search(r'[^\\w]\\s*$', line):\n      break", "response": "Checks if a line contains a redundant virtual function - specifier."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncheck if a line contains a redundant override or final virt - specifier.", "response": "def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error):\n  \"\"\"Check if line contains a redundant \"override\" or \"final\" virt-specifier.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    error: The function to call with any errors found.\n  \"\"\"\n  # Look for closing parenthesis nearby.  We need one to confirm where\n  # the declarator ends and where the virt-specifier starts to avoid\n  # false positives.\n  line = clean_lines.elided[linenum]\n  declarator_end = line.rfind(')')\n  if declarator_end >= 0:\n    fragment = line[declarator_end:]\n  else:\n    if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0:\n      fragment = line\n    else:\n      return\n\n  # Check that at most one of \"override\" or \"final\" is present, not both\n  if Search(r'\\boverride\\b', fragment) and Search(r'\\bfinal\\b', fragment):\n    error(filename, linenum, 'readability/inheritance', 4,\n          ('\"override\" is redundant since function is '\n           'already declared as \"final\"'))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncheck that the new block is in a namespace.", "response": "def IsBlockInNameSpace(nesting_state, is_forward_declaration):\n  \"\"\"Checks that the new block is directly in a namespace.\n\n  Args:\n    nesting_state: The _NestingState object that contains info about our state.\n    is_forward_declaration: If the class is a forward declared class.\n  Returns:\n    Whether or not the new block is directly in a namespace.\n  \"\"\"\n  if is_forward_declaration:\n    return len(nesting_state.stack) >= 1 and (\n      isinstance(nesting_state.stack[-1], _NamespaceInfo))\n\n\n  return (len(nesting_state.stack) > 1 and\n          nesting_state.stack[-1].check_namespace_indentation and\n          isinstance(nesting_state.stack[-2], _NamespaceInfo))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item,\n                                    raw_lines_no_comments, linenum):\n  \"\"\"This method determines if we should apply our namespace indentation check.\n\n  Args:\n    nesting_state: The current nesting state.\n    is_namespace_indent_item: If we just put a new class on the stack, True.\n      If the top of the stack is not a class, or we did not recently\n      add the class, False.\n    raw_lines_no_comments: The lines without the comments.\n    linenum: The current line number we are processing.\n\n  Returns:\n    True if we should apply our namespace indentation check. Currently, it\n    only works for classes and namespaces inside of a namespace.\n  \"\"\"\n\n  is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments,\n                                                     linenum)\n\n  if not (is_namespace_indent_item or is_forward_declaration):\n    return False\n\n  # If we are in a macro, we do not want to check the namespace indentation.\n  if IsMacroDefinition(raw_lines_no_comments, linenum):\n    return False\n\n  return IsBlockInNameSpace(nesting_state, is_forward_declaration)", "response": "This function determines if we should apply our namespace indentation check."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nflagging those C ++ 14 features that we restrict.", "response": "def FlagCxx14Features(filename, clean_lines, linenum, error):\n  \"\"\"Flag those C++14 features that we restrict.\n\n  Args:\n    filename: The name of the current file.\n    clean_lines: A CleansedLines instance containing the file.\n    linenum: The number of the line to check.\n    error: The function to call with any errors found.\n  \"\"\"\n  line = clean_lines.elided[linenum]\n\n  include = Match(r'\\s*#\\s*include\\s+[<\"]([^<\"]+)[\">]', line)\n\n  # Flag unapproved C++14 headers.\n  if include and include.group(1) in ('scoped_allocator', 'shared_mutex'):\n    error(filename, linenum, 'build/c++14', 5,\n          ('<%s> is an unapproved C++14 header.') % include.group(1))"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ProcessFileData(filename, file_extension, lines, error,\n                    extra_check_functions=None):\n  \"\"\"Performs lint checks and reports any errors to the given error function.\n\n  Args:\n    filename: Filename of the file that is being processed.\n    file_extension: The extension (dot not included) of the file.\n    lines: An array of strings, each representing a line of the file, with the\n           last element being empty if the file is terminated with a newline.\n    error: A callable to which errors are reported, which takes 4 arguments:\n           filename, line number, error level, and message\n    extra_check_functions: An array of additional check functions that will be\n                           run on each source line. Each function takes 4\n                           arguments: filename, clean_lines, line, error\n  \"\"\"\n  lines = (['// marker so line numbers and indices both start at 1'] + lines +\n           ['// marker so line numbers end in a known way'])\n\n  include_state = _IncludeState()\n  function_state = _FunctionState()\n  nesting_state = NestingState()\n\n  ResetNolintSuppressions()\n\n  CheckForCopyright(filename, lines, error)\n  ProcessGlobalSuppresions(lines)\n  RemoveMultiLineComments(filename, lines, error)\n  clean_lines = CleansedLines(lines)\n\n  if file_extension in GetHeaderExtensions():\n    CheckForHeaderGuard(filename, clean_lines, error)\n\n  for line in range(clean_lines.NumLines()):\n    ProcessLine(filename, file_extension, clean_lines, line,\n                include_state, function_state, nesting_state, error,\n                extra_check_functions)\n    FlagCxx11Features(filename, clean_lines, line, error)\n  nesting_state.CheckCompletedBlocks(filename, error)\n\n  CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error)\n\n  # Check that the .cc file has included its header if it exists.\n  if _IsSourceExtension(file_extension):\n    CheckHeaderFileIncluded(filename, include_state, error)\n\n  # We check here rather than inside ProcessLine so that we see raw\n  # lines rather than \"cleaned\" lines.\n  CheckForBadCharacters(filename, lines, error)\n\n  CheckForNewlineAtEOF(filename, lines, error)", "response": "Processes the file data and checks for errors."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nloads the configuration files and processes the config overrides.", "response": "def ProcessConfigOverrides(filename):\n  \"\"\" Loads the configuration files and processes the config overrides.\n\n  Args:\n    filename: The name of the file being processed by the linter.\n\n  Returns:\n    False if the current |filename| should not be processed further.\n  \"\"\"\n\n  abs_filename = os.path.abspath(filename)\n  cfg_filters = []\n  keep_looking = True\n  while keep_looking:\n    abs_path, base_name = os.path.split(abs_filename)\n    if not base_name:\n      break  # Reached the root directory.\n\n    cfg_file = os.path.join(abs_path, \"CPPLINT.cfg\")\n    abs_filename = abs_path\n    if not os.path.isfile(cfg_file):\n      continue\n\n    try:\n      with open(cfg_file) as file_handle:\n        for line in file_handle:\n          line, _, _ = line.partition('#')  # Remove comments.\n          if not line.strip():\n            continue\n\n          name, _, val = line.partition('=')\n          name = name.strip()\n          val = val.strip()\n          if name == 'set noparent':\n            keep_looking = False\n          elif name == 'filter':\n            cfg_filters.append(val)\n          elif name == 'exclude_files':\n            # When matching exclude_files pattern, use the base_name of\n            # the current file name or the directory name we are processing.\n            # For example, if we are checking for lint errors in /foo/bar/baz.cc\n            # and we found the .cfg file at /foo/CPPLINT.cfg, then the config\n            # file's \"exclude_files\" filter is meant to be checked against \"bar\"\n            # and not \"baz\" nor \"bar/baz.cc\".\n            if base_name:\n              pattern = re.compile(val)\n              if pattern.match(base_name):\n                _cpplint_state.PrintInfo('Ignoring \"%s\": file excluded by '\n                    '\"%s\". File path component \"%s\" matches pattern \"%s\"\\n' %\n                    (filename, cfg_file, base_name, val))\n                return False\n          elif name == 'linelength':\n            global _line_length\n            try:\n                _line_length = int(val)\n            except ValueError:\n                _cpplint_state.PrintError('Line length must be numeric.')\n          elif name == 'extensions':\n              global _valid_extensions\n              try:\n                  extensions = [ext.strip() for ext in val.split(',')]\n                  _valid_extensions = set(extensions)\n              except ValueError:\n                  sys.stderr.write('Extensions should be a comma-separated list of values;'\n                                   'for example: extensions=hpp,cpp\\n'\n                                   'This could not be parsed: \"%s\"' % (val,))\n          elif name == 'headers':\n              global _header_extensions\n              try:\n                  extensions = [ext.strip() for ext in val.split(',')]\n                  _header_extensions = set(extensions)\n              except ValueError:\n                  sys.stderr.write('Extensions should be a comma-separated list of values;'\n                                   'for example: extensions=hpp,cpp\\n'\n                                   'This could not be parsed: \"%s\"' % (val,))\n          elif name == 'root':\n            global _root\n            _root = val\n          else:\n            _cpplint_state.PrintError(\n                'Invalid configuration option (%s) in file %s\\n' %\n                (name, cfg_file))\n\n    except IOError:\n      _cpplint_state.PrintError(\n          \"Skipping config file '%s': Can't open for reading\\n\" % cfg_file)\n      keep_looking = False\n\n  # Apply all the accumulated filters in reverse order (top-level directory\n  # config options having the least priority).\n  for cfg_filter in reversed(cfg_filters):\n     _AddFilters(cfg_filter)\n\n  return True"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ProcessFile(filename, vlevel, extra_check_functions=None):\n\n  _SetVerboseLevel(vlevel)\n  _BackupFilters()\n\n  if not ProcessConfigOverrides(filename):\n    _RestoreFilters()\n    return\n\n  lf_lines = []\n  crlf_lines = []\n  try:\n    # Support the UNIX convention of using \"-\" for stdin.  Note that\n    # we are not opening the file with universal newline support\n    # (which codecs doesn't support anyway), so the resulting lines do\n    # contain trailing '\\r' characters if we are reading a file that\n    # has CRLF endings.\n    # If after the split a trailing '\\r' is present, it is removed\n    # below.\n    if filename == '-':\n      lines = codecs.StreamReaderWriter(sys.stdin,\n                                        codecs.getreader('utf8'),\n                                        codecs.getwriter('utf8'),\n                                        'replace').read().split('\\n')\n    else:\n      lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\\n')\n\n    # Remove trailing '\\r'.\n    # The -1 accounts for the extra trailing blank line we get from split()\n    for linenum in range(len(lines) - 1):\n      if lines[linenum].endswith('\\r'):\n        lines[linenum] = lines[linenum].rstrip('\\r')\n        crlf_lines.append(linenum + 1)\n      else:\n        lf_lines.append(linenum + 1)\n\n  except IOError:\n    _cpplint_state.PrintError(\n        \"Skipping input '%s': Can't open for reading\\n\" % filename)\n    _RestoreFilters()\n    return\n\n  # Note, if no dot is found, this will give the entire filename as the ext.\n  file_extension = filename[filename.rfind('.') + 1:]\n\n  # When reading from stdin, the extension is unknown, so no cpplint tests\n  # should rely on the extension.\n  if filename != '-' and file_extension not in GetAllExtensions():\n    # bazel 0.5.1> uses four distinct generated files that gives a warning\n    # we suppress the warning for these files\n    bazel_gen_files = set([ \n        \"external/local_config_cc/libtool\",\n        \"external/local_config_cc/make_hashed_objlist.py\", \n        \"external/local_config_cc/wrapped_ar\",\n        \"external/local_config_cc/wrapped_clang\",\n        \"external/local_config_cc/xcrunwrapper.sh\",\n    ])\n    if not filename in bazel_gen_files:\n       _cpplint_state.PrintError('Ignoring %s; not a valid file name '\n                                 '(%s)\\n' % (filename, ', '.join(GetAllExtensions())))\n  else:\n    ProcessFileData(filename, file_extension, lines, Error,\n                    extra_check_functions)\n\n    # If end-of-line sequences are a mix of LF and CR-LF, issue\n    # warnings on the lines with CR.\n    #\n    # Don't issue any warnings if all lines are uniformly LF or CR-LF,\n    # since critique can handle these just fine, and the style guide\n    # doesn't dictate a particular end of line sequence.\n    #\n    # We can't depend on os.linesep to determine what the desired\n    # end-of-line sequence should be, since that will return the\n    # server-side end-of-line sequence.\n    if lf_lines and crlf_lines:\n      # Warn on every line with CR.  An alternative approach might be to\n      # check whether the file is mostly CRLF or just LF, and warn on the\n      # minority, we bias toward LF here since most tools prefer LF.\n      for linenum in crlf_lines:\n        Error(filename, linenum, 'whitespace/newline', 1,\n              'Unexpected \\\\r (^M) found; better to use only \\\\n')\n\n  _RestoreFilters()", "response": "Processes a single file."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nprint a list of all the error - categories used by error messages.", "response": "def PrintCategories():\n  \"\"\"Prints a list of all the error-categories used by error messages.\n\n  These are the categories used to filter messages via --filter.\n  \"\"\"\n  sys.stderr.write(''.join('  %s\\n' % cat for cat in _ERROR_CATEGORIES))\n  sys.exit(0)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef ParseArguments(args):\n  try:\n    (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=',\n                                                 'counting=',\n                                                 'filter=',\n                                                 'root=',\n                                                 'repository=',\n                                                 'linelength=',\n                                                 'extensions=',\n                                                 'exclude=',\n                                                 'headers=',\n                                                 'quiet',\n                                                 'recursive'])\n  except getopt.GetoptError:\n    PrintUsage('Invalid arguments.')\n\n  verbosity = _VerboseLevel()\n  output_format = _OutputFormat()\n  filters = ''\n  counting_style = ''\n  recursive = False\n\n  for (opt, val) in opts:\n    if opt == '--help':\n      PrintUsage(None)\n    elif opt == '--output':\n      if val not in ('emacs', 'vs7', 'eclipse', 'junit'):\n        PrintUsage('The only allowed output formats are emacs, vs7, eclipse '\n                   'and junit.')\n      output_format = val\n    elif opt == '--verbose':\n      verbosity = int(val)\n    elif opt == '--filter':\n      filters = val\n      if not filters:\n        PrintCategories()\n    elif opt == '--counting':\n      if val not in ('total', 'toplevel', 'detailed'):\n        PrintUsage('Valid counting options are total, toplevel, and detailed')\n      counting_style = val\n    elif opt == '--root':\n      global _root\n      _root = val\n    elif opt == '--repository':\n      global _repository\n      _repository = val\n    elif opt == '--linelength':\n      global _line_length\n      try:\n        _line_length = int(val)\n      except ValueError:\n        PrintUsage('Line length must be digits.')\n    elif opt == '--exclude':\n      global _excludes\n      if not _excludes:\n        _excludes = set()\n      _excludes.update(glob.glob(val))\n    elif opt == '--extensions':\n      global _valid_extensions\n      try:\n        _valid_extensions = set(val.split(','))\n      except ValueError:\n          PrintUsage('Extensions must be comma seperated list.')\n    elif opt == '--headers':\n      global _header_extensions\n      try:\n          _header_extensions = set(val.split(','))\n      except ValueError:\n        PrintUsage('Extensions must be comma seperated list.')\n    elif opt == '--recursive':\n      recursive = True\n    elif opt == '--quiet':\n      global _quiet\n      _quiet = True\n\n  if not filenames:\n    PrintUsage('No files were specified.')\n\n  if recursive:\n    filenames = _ExpandDirectories(filenames)\n\n  if _excludes:\n    filenames = _FilterExcludedFiles(filenames)\n\n  _SetOutputFormat(output_format)\n  _SetVerboseLevel(verbosity)\n  _SetFilters(filters)\n  _SetCountingStyle(counting_style)\n\n  return filenames", "response": "Parses the command line arguments and sets the output format and verbosity level as side - effectss."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _ExpandDirectories(filenames):\n  expanded = set()\n  for filename in filenames:\n      if not os.path.isdir(filename):\n        expanded.add(filename)\n        continue\n\n      for root, _, files in os.walk(filename):\n        for loopfile in files:\n          fullname = os.path.join(root, loopfile)\n          if fullname.startswith('.' + os.path.sep):\n            fullname = fullname[len('.' + os.path.sep):]\n          expanded.add(fullname)\n\n  filtered = []\n  for filename in expanded:\n      if os.path.splitext(filename)[1][1:] in GetAllExtensions():\n          filtered.append(filename)\n\n  return filtered", "response": "Searches a list of filenames and replaces directories with their full names."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nfiltering out files listed in the exclude command line switch.", "response": "def _FilterExcludedFiles(filenames):\n  \"\"\"Filters out files listed in the --exclude command line switch. File paths\n  in the switch are evaluated relative to the current working directory\n  \"\"\"\n  exclude_paths = [os.path.abspath(f) for f in _excludes]\n  return [f for f in filenames if os.path.abspath(f) not in exclude_paths]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the line number of the previous occurrence of a header.", "response": "def FindHeader(self, header):\n    \"\"\"Check if a header has already been included.\n\n    Args:\n      header: header to check.\n    Returns:\n      Line number of previous occurrence, or -1 if the header has not\n      been seen before.\n    \"\"\"\n    for section_list in self.include_list:\n      for f in section_list:\n        if f[0] == header:\n          return f[1]\n    return -1"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nresets the section checking for preprocessor directive.", "response": "def ResetSection(self, directive):\n    \"\"\"Reset section checking for preprocessor directive.\n\n    Args:\n      directive: preprocessor directive (e.g. \"if\", \"else\").\n    \"\"\"\n    # The name of the current section.\n    self._section = self._INITIAL_SECTION\n    # The path of last found header.\n    self._last_header = ''\n\n    # Update list of includes.  Note that we never pop from the\n    # include list.\n    if directive in ('if', 'ifdef', 'ifndef'):\n      self.include_list.append([])\n    elif directive in ('else', 'elif'):\n      self.include_list[-1] = []"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef IsInAlphabeticalOrder(self, clean_lines, linenum, header_path):\n    # If previous section is different from current section, _last_header will\n    # be reset to empty string, so it's always less than current header.\n    #\n    # If previous line was a blank line, assume that the headers are\n    # intentionally sorted the way they are.\n    if (self._last_header > header_path and\n        Match(r'^\\s*#\\s*include\\b', clean_lines.elided[linenum - 1])):\n      return False\n    return True", "response": "Checks if a header is in alphabetical order."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CheckNextIncludeOrder(self, header_type):\n    error_message = ('Found %s after %s' %\n                     (self._TYPE_NAMES[header_type],\n                      self._SECTION_NAMES[self._section]))\n\n    last_section = self._section\n\n    if header_type == _C_SYS_HEADER:\n      if self._section <= self._C_SECTION:\n        self._section = self._C_SECTION\n      else:\n        self._last_header = ''\n        return error_message\n    elif header_type == _CPP_SYS_HEADER:\n      if self._section <= self._CPP_SECTION:\n        self._section = self._CPP_SECTION\n      else:\n        self._last_header = ''\n        return error_message\n    elif header_type == _LIKELY_MY_HEADER:\n      if self._section <= self._MY_H_SECTION:\n        self._section = self._MY_H_SECTION\n      else:\n        self._section = self._OTHER_H_SECTION\n    elif header_type == _POSSIBLE_MY_HEADER:\n      if self._section <= self._MY_H_SECTION:\n        self._section = self._MY_H_SECTION\n      else:\n        # This will always be the fallback because we're not sure\n        # enough that the header is associated with this file.\n        self._section = self._OTHER_H_SECTION\n    else:\n      assert header_type == _OTHER_HEADER\n      self._section = self._OTHER_H_SECTION\n\n    if last_section != self._section:\n      self._last_header = ''\n\n    return ''", "response": "Checks if the next include order of the given header type is in the right order."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets the module s verbosity.", "response": "def SetVerboseLevel(self, level):\n    \"\"\"Sets the module's verbosity, and returns the previous setting.\"\"\"\n    last_verbose_level = self.verbose_level\n    self.verbose_level = level\n    return last_verbose_level"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadd more filters to the existing list of error - message filters.", "response": "def AddFilters(self, filters):\n    \"\"\" Adds more filters to the existing list of error-message filters. \"\"\"\n    for filt in filters.split(','):\n      clean_filt = filt.strip()\n      if clean_filt:\n        self.filters.append(clean_filt)\n    for filt in self.filters:\n      if not (filt.startswith('+') or filt.startswith('-')):\n        raise ValueError('Every filter in --filters must start with + or -'\n                         ' (%s does not)' % filt)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nincrement the module s error statistic.", "response": "def IncrementErrorCount(self, category):\n    \"\"\"Bumps the module's error statistic.\"\"\"\n    self.error_count += 1\n    if self.counting in ('toplevel', 'detailed'):\n      if self.counting != 'detailed':\n        category = category.split('/')[0]\n      if category not in self.errors_by_category:\n        self.errors_by_category[category] = 0\n      self.errors_by_category[category] += 1"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef PrintErrorCounts(self):\n    for category, count in sorted(iteritems(self.errors_by_category)):\n      self.PrintInfo('Category \\'%s\\' errors found: %d\\n' %\n                       (category, count))\n    if self.error_count > 0:\n      self.PrintInfo('Total errors found: %d\\n' % self.error_count)", "response": "Print a summary of errors by category and the total."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef Begin(self, function_name):\n    self.in_a_function = True\n    self.lines_in_function = 0\n    self.current_function = function_name", "response": "Starts analyzing the function body."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef RepositoryName(self):\n    fullname = self.FullName()\n\n    if os.path.exists(fullname):\n      project_dir = os.path.dirname(fullname)\n\n      # If the user specified a repository path, it exists, and the file is\n      # contained in it, use the specified repository path\n      if _repository:\n        repo = FileInfo(_repository).FullName()\n        root_dir = project_dir\n        while os.path.exists(root_dir):\n          # allow case insensitive compare on Windows\n          if os.path.normcase(root_dir) == os.path.normcase(repo):\n            return os.path.relpath(fullname, root_dir).replace('\\\\', '/')\n          one_up_dir = os.path.dirname(root_dir)\n          if one_up_dir == root_dir:\n            break\n          root_dir = one_up_dir\n\n      if os.path.exists(os.path.join(project_dir, \".svn\")):\n        # If there's a .svn file in the current directory, we recursively look\n        # up the directory tree for the top of the SVN checkout\n        root_dir = project_dir\n        one_up_dir = os.path.dirname(root_dir)\n        while os.path.exists(os.path.join(one_up_dir, \".svn\")):\n          root_dir = os.path.dirname(root_dir)\n          one_up_dir = os.path.dirname(one_up_dir)\n\n        prefix = os.path.commonprefix([root_dir, project_dir])\n        return fullname[len(prefix) + 1:]\n\n      # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by\n      # searching up from the current path.\n      root_dir = current_dir = os.path.dirname(fullname)\n      while current_dir != os.path.dirname(current_dir):\n        if (os.path.exists(os.path.join(current_dir, \".git\")) or\n            os.path.exists(os.path.join(current_dir, \".hg\")) or\n            os.path.exists(os.path.join(current_dir, \".svn\"))):\n          root_dir = current_dir\n        current_dir = os.path.dirname(current_dir)\n\n      if (os.path.exists(os.path.join(root_dir, \".git\")) or\n          os.path.exists(os.path.join(root_dir, \".hg\")) or\n          os.path.exists(os.path.join(root_dir, \".svn\"))):\n        prefix = os.path.commonprefix([root_dir, project_dir])\n        return fullname[len(prefix) + 1:]\n\n    # Don't know what to do; header guard warnings may be wrong...\n    return fullname", "response": "r Returns the full path name of the current repository."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsplit the file into the directory basename and extension.", "response": "def Split(self):\n    \"\"\"Splits the file into the directory, basename, and extension.\n\n    For 'chrome/browser/browser.cc', Split() would\n    return ('chrome/browser', 'browser', '.cc')\n\n    Returns:\n      A tuple of (directory, basename, extension).\n    \"\"\"\n\n    googlename = self.RepositoryName()\n    project, rest = os.path.split(googlename)\n    return (project,) + os.path.splitext(rest)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _CollapseStrings(elided):\n    if _RE_PATTERN_INCLUDE.match(elided):\n      return elided\n\n    # Remove escaped characters first to make quote/single quote collapsing\n    # basic.  Things that look like escaped characters shouldn't occur\n    # outside of strings and chars.\n    elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided)\n\n    # Replace quoted strings and digit separators.  Both single quotes\n    # and double quotes are processed in the same loop, otherwise\n    # nested quotes wouldn't work.\n    collapsed = ''\n    while True:\n      # Find the first quote character\n      match = Match(r'^([^\\'\"]*)([\\'\"])(.*)$', elided)\n      if not match:\n        collapsed += elided\n        break\n      head, quote, tail = match.groups()\n\n      if quote == '\"':\n        # Collapse double quoted strings\n        second_quote = tail.find('\"')\n        if second_quote >= 0:\n          collapsed += head + '\"\"'\n          elided = tail[second_quote + 1:]\n        else:\n          # Unmatched double quote, don't bother processing the rest\n          # of the line since this is probably a multiline string.\n          collapsed += elided\n          break\n      else:\n        # Found single quote, check nearby text to eliminate digit separators.\n        #\n        # There is no special handling for floating point here, because\n        # the integer/fractional/exponent parts would all be parsed\n        # correctly as long as there are digits on both sides of the\n        # separator.  So we are fine as long as we don't see something\n        # like \"0.'3\" (gcc 4.9.0 will not allow this literal).\n        if Search(r'\\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head):\n          match_literal = Match(r'^((?:\\'?[0-9a-zA-Z_])*)(.*)$', \"'\" + tail)\n          collapsed += head + match_literal.group(1).replace(\"'\", '')\n          elided = match_literal.group(2)\n        else:\n          second_quote = tail.find('\\'')\n          if second_quote >= 0:\n            collapsed += head + \"''\"\n            elided = tail[second_quote + 1:]\n          else:\n            # Unmatched single quote\n            collapsed += elided\n            break\n\n    return collapsed", "response": "Collapse a string and chars on a line to a simple \"\" or '' blocks."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nchecks if the end of a namespace comment is correct.", "response": "def CheckEnd(self, filename, clean_lines, linenum, error):\n    \"\"\"Check end of namespace comments.\"\"\"\n    line = clean_lines.raw_lines[linenum]\n\n    # Check how many lines is enclosed in this namespace.  Don't issue\n    # warning for missing namespace comments if there aren't enough\n    # lines.  However, do apply checks if there is already an end of\n    # namespace comment and it's incorrect.\n    #\n    # TODO(unknown): We always want to check end of namespace comments\n    # if a namespace is large, but sometimes we also want to apply the\n    # check if a short namespace contained nontrivial things (something\n    # other than forward declarations).  There is currently no logic on\n    # deciding what these nontrivial things are, so this check is\n    # triggered by namespace size only, which works most of the time.\n    if (linenum - self.starting_linenum < 10\n        and not Match(r'^\\s*};*\\s*(//|/\\*).*\\bnamespace\\b', line)):\n      return\n\n    # Look for matching comment at end of namespace.\n    #\n    # Note that we accept C style \"/* */\" comments for terminating\n    # namespaces, so that code that terminate namespaces inside\n    # preprocessor macros can be cpplint clean.\n    #\n    # We also accept stuff like \"// end of namespace <name>.\" with the\n    # period at the end.\n    #\n    # Besides these, we don't accept anything else, otherwise we might\n    # get false negatives when existing comment is a substring of the\n    # expected namespace.\n    if self.name:\n      # Named namespace\n      if not Match((r'^\\s*};*\\s*(//|/\\*).*\\bnamespace\\s+' +\n                    re.escape(self.name) + r'[\\*/\\.\\\\\\s]*$'),\n                   line):\n        error(filename, linenum, 'readability/namespace', 5,\n              'Namespace should be terminated with \"// namespace %s\"' %\n              self.name)\n    else:\n      # Anonymous namespace\n      if not Match(r'^\\s*};*\\s*(//|/\\*).*\\bnamespace[\\*/\\.\\\\\\s]*$', line):\n        # If \"// namespace anonymous\" or \"// anonymous namespace (more text)\",\n        # mention \"// anonymous namespace\" as an acceptable form\n        if Match(r'^\\s*}.*\\b(namespace anonymous|anonymous namespace)\\b', line):\n          error(filename, linenum, 'readability/namespace', 5,\n                'Anonymous namespace should be terminated with \"// namespace\"'\n                ' or \"// anonymous namespace\"')\n        else:\n          error(filename, linenum, 'readability/namespace', 5,\n                'Anonymous namespace should be terminated with \"// namespace\"')"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncheck if the current position is inside template argument list.", "response": "def InTemplateArgumentList(self, clean_lines, linenum, pos):\n    \"\"\"Check if current position is inside template argument list.\n\n    Args:\n      clean_lines: A CleansedLines instance containing the file.\n      linenum: The number of the line to check.\n      pos: position just after the suspected template argument.\n    Returns:\n      True if (linenum, pos) is inside template arguments.\n    \"\"\"\n    while linenum < clean_lines.NumLines():\n      # Find the earliest character that might indicate a template argument\n      line = clean_lines.elided[linenum]\n      match = Match(r'^[^{};=\\[\\]\\.<>]*(.)', line[pos:])\n      if not match:\n        linenum += 1\n        pos = 0\n        continue\n      token = match.group(1)\n      pos += len(match.group(0))\n\n      # These things do not look like template argument list:\n      #   class Suspect {\n      #   class Suspect x; }\n      if token in ('{', '}', ';'): return False\n\n      # These things look like template argument list:\n      #   template <class Suspect>\n      #   template <class Suspect = default_value>\n      #   template <class Suspect[]>\n      #   template <class Suspect...>\n      if token in ('>', '=', '[', ']', '.'): return True\n\n      # Check if token is an unmatched '<'.\n      # If not, move on to the next character.\n      if token != '<':\n        pos += 1\n        if pos >= len(line):\n          linenum += 1\n          pos = 0\n        continue\n\n      # We can't be sure if we just find a single '<', and need to\n      # find the matching '>'.\n      (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - 1)\n      if end_pos < 0:\n        # Not sure if template argument list or syntax error in file\n        return False\n      linenum = end_line\n      pos = end_pos\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nupdate the preprocessor stack.", "response": "def UpdatePreprocessor(self, line):\n    \"\"\"Update preprocessor stack.\n\n    We need to handle preprocessors due to classes like this:\n      #ifdef SWIG\n      struct ResultDetailsPageElementExtensionPoint {\n      #else\n      struct ResultDetailsPageElementExtensionPoint : public Extension {\n      #endif\n\n    We make the following assumptions (good enough for most files):\n    - Preprocessor condition evaluates to true from #if up to first\n      #else/#elif/#endif.\n\n    - Preprocessor condition evaluates to false from #else/#elif up\n      to #endif.  We still perform lint checks on these lines, but\n      these do not affect nesting stack.\n\n    Args:\n      line: current line to check.\n    \"\"\"\n    if Match(r'^\\s*#\\s*(if|ifdef|ifndef)\\b', line):\n      # Beginning of #if block, save the nesting stack here.  The saved\n      # stack will allow us to restore the parsing state in the #else case.\n      self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack)))\n    elif Match(r'^\\s*#\\s*(else|elif)\\b', line):\n      # Beginning of #else block\n      if self.pp_stack:\n        if not self.pp_stack[-1].seen_else:\n          # This is the first #else or #elif block.  Remember the\n          # whole nesting stack up to this point.  This is what we\n          # keep after the #endif.\n          self.pp_stack[-1].seen_else = True\n          self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack)\n\n        # Restore the stack to how it was before the #if\n        self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if)\n      else:\n        # TODO(unknown): unexpected #else, issue warning?\n        pass\n    elif Match(r'^\\s*#\\s*endif\\b', line):\n      # End of #if or #else blocks.\n      if self.pp_stack:\n        # If we saw an #else, we will need to restore the nesting\n        # stack to its former state before the #else, otherwise we\n        # will just continue from where we left off.\n        if self.pp_stack[-1].seen_else:\n          # Here we can just use a shallow copy since we are the last\n          # reference to it.\n          self.stack = self.pp_stack[-1].stack_before_else\n        # Drop the corresponding #if\n        self.pp_stack.pop()\n      else:\n        # TODO(unknown): unexpected #endif, issue warning?\n        pass"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef Update(self, filename, clean_lines, linenum, error):\n    line = clean_lines.elided[linenum]\n\n    # Remember top of the previous nesting stack.\n    #\n    # The stack is always pushed/popped and not modified in place, so\n    # we can just do a shallow copy instead of copy.deepcopy.  Using\n    # deepcopy would slow down cpplint by ~28%.\n    if self.stack:\n      self.previous_stack_top = self.stack[-1]\n    else:\n      self.previous_stack_top = None\n\n    # Update pp_stack\n    self.UpdatePreprocessor(line)\n\n    # Count parentheses.  This is to avoid adding struct arguments to\n    # the nesting stack.\n    if self.stack:\n      inner_block = self.stack[-1]\n      depth_change = line.count('(') - line.count(')')\n      inner_block.open_parentheses += depth_change\n\n      # Also check if we are starting or ending an inline assembly block.\n      if inner_block.inline_asm in (_NO_ASM, _END_ASM):\n        if (depth_change != 0 and\n            inner_block.open_parentheses == 1 and\n            _MATCH_ASM.match(line)):\n          # Enter assembly block\n          inner_block.inline_asm = _INSIDE_ASM\n        else:\n          # Not entering assembly block.  If previous line was _END_ASM,\n          # we will now shift to _NO_ASM state.\n          inner_block.inline_asm = _NO_ASM\n      elif (inner_block.inline_asm == _INSIDE_ASM and\n            inner_block.open_parentheses == 0):\n        # Exit assembly block\n        inner_block.inline_asm = _END_ASM\n\n    # Consume namespace declaration at the beginning of the line.  Do\n    # this in a loop so that we catch same line declarations like this:\n    #   namespace proto2 { namespace bridge { class MessageSet; } }\n    while True:\n      # Match start of namespace.  The \"\\b\\s*\" below catches namespace\n      # declarations even if it weren't followed by a whitespace, this\n      # is so that we don't confuse our namespace checker.  The\n      # missing spaces will be flagged by CheckSpacing.\n      namespace_decl_match = Match(r'^\\s*namespace\\b\\s*([:\\w]+)?(.*)$', line)\n      if not namespace_decl_match:\n        break\n\n      new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum)\n      self.stack.append(new_namespace)\n\n      line = namespace_decl_match.group(2)\n      if line.find('{') != -1:\n        new_namespace.seen_open_brace = True\n        line = line[line.find('{') + 1:]\n\n    # Look for a class declaration in whatever is left of the line\n    # after parsing namespaces.  The regexp accounts for decorated classes\n    # such as in:\n    #   class LOCKABLE API Object {\n    #   };\n    class_decl_match = Match(\n        r'^(\\s*(?:template\\s*<[\\w\\s<>,:=]*>\\s*)?'\n        r'(class|struct)\\s+(?:[A-Z_]+\\s+)*(\\w+(?:::\\w+)*))'\n        r'(.*)$', line)\n    if (class_decl_match and\n        (not self.stack or self.stack[-1].open_parentheses == 0)):\n      # We do not want to accept classes that are actually template arguments:\n      #   template <class Ignore1,\n      #             class Ignore2 = Default<Args>,\n      #             template <Args> class Ignore3>\n      #   void Function() {};\n      #\n      # To avoid template argument cases, we scan forward and look for\n      # an unmatched '>'.  If we see one, assume we are inside a\n      # template argument list.\n      end_declaration = len(class_decl_match.group(1))\n      if not self.InTemplateArgumentList(clean_lines, linenum, end_declaration):\n        self.stack.append(_ClassInfo(\n            class_decl_match.group(3), class_decl_match.group(2),\n            clean_lines, linenum))\n        line = class_decl_match.group(4)\n\n    # If we have not yet seen the opening brace for the innermost block,\n    # run checks here.\n    if not self.SeenOpenBrace():\n      self.stack[-1].CheckBegin(filename, clean_lines, linenum, error)\n\n    # Update access control if we are inside a class/struct\n    if self.stack and isinstance(self.stack[-1], _ClassInfo):\n      classinfo = self.stack[-1]\n      access_match = Match(\n          r'^(.*)\\b(public|private|protected|signals)(\\s+(?:slots\\s*)?)?'\n          r':(?:[^:]|$)',\n          line)\n      if access_match:\n        classinfo.access = access_match.group(2)\n\n        # Check that access keywords are indented +1 space.  Skip this\n        # check if the keywords are not preceded by whitespaces.\n        indent = access_match.group(1)\n        if (len(indent) != classinfo.class_indent + 1 and\n            Match(r'^\\s*$', indent)):\n          if classinfo.is_struct:\n            parent = 'struct ' + classinfo.name\n          else:\n            parent = 'class ' + classinfo.name\n          slots = ''\n          if access_match.group(3):\n            slots = access_match.group(3)\n          error(filename, linenum, 'whitespace/indent', 3,\n                '%s%s: should be indented +1 space inside %s' % (\n                    access_match.group(2), slots, parent))\n\n    # Consume braces or semicolons from what's left of the line\n    while True:\n      # Match first brace, semicolon, or closed parenthesis.\n      matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line)\n      if not matched:\n        break\n\n      token = matched.group(1)\n      if token == '{':\n        # If namespace or class hasn't seen a opening brace yet, mark\n        # namespace/class head as complete.  Push a new block onto the\n        # stack otherwise.\n        if not self.SeenOpenBrace():\n          self.stack[-1].seen_open_brace = True\n        elif Match(r'^extern\\s*\"[^\"]*\"\\s*\\{', line):\n          self.stack.append(_ExternCInfo(linenum))\n        else:\n          self.stack.append(_BlockInfo(linenum, True))\n          if _MATCH_ASM.match(line):\n            self.stack[-1].inline_asm = _BLOCK_ASM\n\n      elif token == ';' or token == ')':\n        # If we haven't seen an opening brace yet, but we already saw\n        # a semicolon, this is probably a forward declaration.  Pop\n        # the stack for these.\n        #\n        # Similarly, if we haven't seen an opening brace yet, but we\n        # already saw a closing parenthesis, then these are probably\n        # function arguments with extra \"class\" or \"struct\" keywords.\n        # Also pop these stack for these.\n        if not self.SeenOpenBrace():\n          self.stack.pop()\n      else:  # token == '}'\n        # Perform end of block checks and pop the stack.\n        if self.stack:\n          self.stack[-1].CheckEnd(filename, clean_lines, linenum, error)\n          self.stack.pop()\n      line = matched.group(2)", "response": "Updates the nesting state with the current line."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef InnermostClass(self):\n    for i in range(len(self.stack), 0, -1):\n      classinfo = self.stack[i - 1]\n      if isinstance(classinfo, _ClassInfo):\n        return classinfo\n    return None", "response": "Returns the _ClassInfo object if we are inside a class."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking that all classes and namespaces have been completely parsed.", "response": "def CheckCompletedBlocks(self, filename, error):\n    \"\"\"Checks that all classes and namespaces have been completely parsed.\n\n    Call this when all lines in a file have been processed.\n    Args:\n      filename: The name of the current file.\n      error: The function to call with any errors found.\n    \"\"\"\n    # Note: This test can result in false positives if #ifdef constructs\n    # get in the way of brace matching. See the testBuildClass test in\n    # cpplint_unittest.py for an example of this.\n    for obj in self.stack:\n      if isinstance(obj, _ClassInfo):\n        error(filename, obj.starting_linenum, 'build/class', 5,\n              'Failed to find complete declaration of class %s' %\n              obj.name)\n      elif isinstance(obj, _NamespaceInfo):\n        error(filename, obj.starting_linenum, 'build/namespaces', 5,\n              'Failed to find complete declaration of namespace %s' %\n              obj.name)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a new Streamlet by applying map_function to each element of this Streamlet.", "response": "def map(self, map_function):\n    \"\"\"Return a new Streamlet by applying map_function to each element of this Streamlet.\n    \"\"\"\n    from heronpy.streamlet.impl.mapbolt import MapStreamlet\n    map_streamlet = MapStreamlet(map_function, self)\n    self._add_child(map_streamlet)\n    return map_streamlet"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn a new Streamlet by applying map_function to each element of this Streamlet and flattening the result.", "response": "def flat_map(self, flatmap_function):\n    \"\"\"Return a new Streamlet by applying map_function to each element of this Streamlet\n       and flattening the result\n    \"\"\"\n    from heronpy.streamlet.impl.flatmapbolt import FlatMapStreamlet\n    fm_streamlet = FlatMapStreamlet(flatmap_function, self)\n    self._add_child(fm_streamlet)\n    return fm_streamlet"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef filter(self, filter_function):\n    from heronpy.streamlet.impl.filterbolt import FilterStreamlet\n    filter_streamlet = FilterStreamlet(filter_function, self)\n    self._add_child(filter_streamlet)\n    return filter_streamlet", "response": "Return a new Streamlet containing only the elements that satisfy filter_function"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a new Streamlet containing all elements of this streamlet but having num_partitions partitions.", "response": "def repartition(self, num_partitions, repartition_function=None):\n    \"\"\"Return a new Streamlet containing all elements of the this streamlet but having\n    num_partitions partitions. Note that this is different from num_partitions(n) in\n    that new streamlet will be created by the repartition call.\n    If repartiton_function is not None, it is used to decide which parititons\n    (from 0 to num_partitions -1), it should route each element to.\n    It could also return a list of partitions if it wants to send it to multiple\n    partitions.\n    \"\"\"\n    from heronpy.streamlet.impl.repartitionbolt import RepartitionStreamlet\n    if repartition_function is None:\n      repartition_function = lambda x: x\n    repartition_streamlet = RepartitionStreamlet(num_partitions, repartition_function, self)\n    self._add_child(repartition_streamlet)\n    return repartition_streamlet"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef clone(self, num_clones):\n    retval = []\n    for i in range(num_clones):\n      retval.append(self.repartition(self.get_num_partitions()))\n    return retval", "response": "Return num_clones number of streamlets each containing all elements\n    of the current streamlet"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a new Streamlet in which each element of this Streamlet is collected by a window defined by window_config and then reduced using reduce_function.", "response": "def reduce_by_window(self, window_config, reduce_function):\n    \"\"\"Return a new Streamlet in which each element of this Streamlet are collected\n      over a window defined by window_config and then reduced using the reduce_function\n      reduce_function takes two element at one time and reduces them to one element that\n      is used in the subsequent operations.\n    \"\"\"\n    from heronpy.streamlet.impl.reducebywindowbolt import ReduceByWindowStreamlet\n    reduce_streamlet = ReduceByWindowStreamlet(window_config, reduce_function, self)\n    self._add_child(reduce_streamlet)\n    return reduce_streamlet"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef union(self, other_streamlet):\n    from heronpy.streamlet.impl.unionbolt import UnionStreamlet\n    union_streamlet = UnionStreamlet(self, other_streamlet)\n    self._add_child(union_streamlet)\n    other_streamlet._add_child(union_streamlet)\n    return union_streamlet", "response": "Returns a new Streamlet that consists of elements of both this and other_streamlet."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef transform(self, transform_operator):\n    from heronpy.streamlet.impl.transformbolt import TransformStreamlet\n    transform_streamlet = TransformStreamlet(transform_operator, self)\n    self._add_child(transform_streamlet)\n    return transform_streamlet", "response": "Returns a new Streamlet by applying the transform_operator on each element of this Streamlet."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nlogs all elements of this streamlet. This returns nothing.", "response": "def log(self):\n    \"\"\"Logs all elements of this streamlet. This returns nothing\n    \"\"\"\n    from heronpy.streamlet.impl.logbolt import LogStreamlet\n    log_streamlet = LogStreamlet(self)\n    self._add_child(log_streamlet)\n    return"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncall consume_function for each element of this streamlet. This function returns nothing.", "response": "def consume(self, consume_function):\n    \"\"\"Calls consume_function for each element of this streamlet. This function returns nothing\n    \"\"\"\n    from heronpy.streamlet.impl.consumebolt import ConsumeStreamlet\n    consume_streamlet = ConsumeStreamlet(consume_function, self)\n    self._add_child(consume_streamlet)\n    return"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a new Streamlet by joining this streamlet with the given join_streamlet", "response": "def join(self, join_streamlet, window_config, join_function):\n    \"\"\"Return a new Streamlet by joining join_streamlet with this streamlet\n    \"\"\"\n    from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt\n    join_streamlet_result = JoinStreamlet(JoinBolt.INNER, window_config,\n                                          join_function, self, join_streamlet)\n    self._add_child(join_streamlet_result)\n    join_streamlet._add_child(join_streamlet_result)\n    return join_streamlet_result"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a new Streamlet by outer right join with this streamlet", "response": "def outer_right_join(self, join_streamlet, window_config, join_function):\n    \"\"\"Return a new Streamlet by outer right join_streamlet with this streamlet\n    \"\"\"\n    from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt\n    join_streamlet_result = JoinStreamlet(JoinBolt.OUTER_RIGHT, window_config,\n                                          join_function, self, join_streamlet)\n    self._add_child(join_streamlet_result)\n    join_streamlet._add_child(join_streamlet_result)\n    return join_streamlet_result"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef outer_left_join(self, join_streamlet, window_config, join_function):\n    from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt\n    join_streamlet_result = JoinStreamlet(JoinBolt.OUTER_LEFT, window_config,\n                                          join_function, self, join_streamlet)\n    self._add_child(join_streamlet_result)\n    join_streamlet._add_child(join_streamlet_result)\n    return join_streamlet_result", "response": "Return a new Streamlet by left join_streamlet with this streamlet"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a new Streamlet by outer join_streamlet with this streamlet", "response": "def outer_join(self, join_streamlet, window_config, join_function):\n    \"\"\"Return a new Streamlet by outer join_streamlet with this streamlet\n    \"\"\"\n    from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt\n\n    join_streamlet_result = JoinStreamlet(JoinBolt.OUTER, window_config,\n                                          join_function, self, join_streamlet)\n    self._add_child(join_streamlet_result)\n    join_streamlet._add_child(join_streamlet_result)\n    return join_streamlet_result"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef reduce_by_key_and_window(self, window_config, reduce_function):\n    from heronpy.streamlet.impl.reducebykeyandwindowbolt import ReduceByKeyAndWindowStreamlet\n    reduce_streamlet = ReduceByKeyAndWindowStreamlet(window_config, reduce_function, self)\n    self._add_child(reduce_streamlet)\n    return reduce_streamlet", "response": "Return a new Streamlet in which each key value pair of this Streamlet is collected by the reduce_function and the time_window is calculated over the time_window."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get(self):\n    clusters = self.get_arguments(constants.PARAM_CLUSTER)\n    environs = self.get_arguments(constants.PARAM_ENVIRON)\n    topology_names = self.get_arguments(constants.PARAM_TOPOLOGY)\n\n    ret = {}\n\n    if len(topology_names) > 1:\n      if not clusters:\n        message = \"Missing argument\" + constants.PARAM_CLUSTER\n        self.write_error_response(message)\n        return\n\n      if not environs:\n        message = \"Missing argument\" + constants.PARAM_ENVIRON\n        self.write_error_response(message)\n        return\n\n    ret = {}\n    topologies = self.tracker.topologies\n    for topology in topologies:\n      cluster = topology.cluster\n      environ = topology.environ\n      topology_name = topology.name\n      if not cluster or not environ:\n        continue\n\n      # This cluster is not asked for.\n      if clusters and cluster not in clusters:\n        continue\n\n      # This environ is not asked for.\n      if environs and environ not in environs:\n        continue\n\n      if topology_names and topology_name not in topology_names:\n        continue\n\n      if cluster not in ret:\n        ret[cluster] = {}\n      if environ not in ret[cluster]:\n        ret[cluster][environ] = {}\n      ret[cluster][environ][topology_name] = topology.get_machines()\n\n    self.write_success_response(ret)", "response": "get method returns a list of dicts"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nstream and handle the output of a process", "response": "def _async_stream_process_output(process, stream_fn, handler):\n  \"\"\" Stream and handle the output of a process\n  :param process: the process to stream the output for\n  :param stream_fn: the function that applies handler to process\n  :param handler: a function that will be called for each log line\n  :return: None\n  \"\"\"\n  logging_thread = Thread(target=stream_fn, args=(process, handler, ))\n\n  # Setting the logging thread as a daemon thread will allow it to exit with the program\n  # rather than blocking the exit waiting for it to be handled manually.\n  logging_thread.daemon = True\n  logging_thread.start()\n\n  return logging_thread"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngetting the topology jars Returns the first file in the directory that matches the pattern.", "response": "def pick(dirname, pattern):\n  '''\n  Get the topology jars\n  :param dirname:\n  :param pattern:\n  :return:\n  '''\n  file_list = fnmatch.filter(os.listdir(dirname), pattern)\n  return file_list[0] if file_list else None"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get(self):\n    try:\n      cluster = self.get_argument_cluster()\n      role = self.get_argument_role()\n      environ = self.get_argument_environ()\n      topology_name = self.get_argument_topology()\n      component = self.get_argument_component()\n      metric_names = self.get_required_arguments_metricnames()\n      start_time = self.get_argument_starttime()\n      end_time = self.get_argument_endtime()\n      self.validateInterval(start_time, end_time)\n      instances = self.get_arguments(constants.PARAM_INSTANCE)\n\n      topology = self.tracker.getTopologyByClusterRoleEnvironAndName(\n          cluster, role, environ, topology_name)\n      metrics = yield tornado.gen.Task(metricstimeline.getMetricsTimeline,\n                                       topology.tmaster, component, metric_names,\n                                       instances, int(start_time), int(end_time))\n      self.write_success_response(metrics)\n    except Exception as e:\n      Log.debug(traceback.format_exc())\n      self.write_error_response(e)", "response": "get method - gets the metrics from the server"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_parser():\n  help_epilog = '''Getting more help:\n  heron-explorer help <command>     Disply help and options for <command>\\n\n  For detailed documentation, go to http://heronstreaming.io'''\n\n  parser = argparse.ArgumentParser(\n      prog='heron-explorer',\n      epilog=help_epilog,\n      formatter_class=SubcommandHelpFormatter,\n      add_help=False)\n\n  # sub-commands\n  subparsers = parser.add_subparsers(\n      title=\"Available commands\",\n      metavar='<command> <options>')\n\n  # subparser for subcommands related to clusters\n  clusters.create_parser(subparsers)\n\n  # subparser for subcommands related to logical plan\n  logicalplan.create_parser(subparsers)\n\n  # subparser for subcommands related to physical plan\n  physicalplan.create_parser(subparsers)\n\n  # subparser for subcommands related to displaying info\n  topologies.create_parser(subparsers)\n\n  # subparser for help subcommand\n  help.create_parser(subparsers)\n\n  # subparser for version subcommand\n  version.create_parser(subparsers)\n\n  return parser", "response": "create parser for all available commands"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef expand_args(command):\n\n    # Prepare arguments.\n    if isinstance(command, (str, unicode)):\n        splitter = shlex.shlex(command.encode('utf-8'))\n        splitter.whitespace = '|'\n        splitter.whitespace_split = True\n        command = []\n\n        while True:\n            token = splitter.get_token()\n            if token:\n                command.append(token)\n            else:\n                break\n\n        command = list(map(shlex.split, command))\n\n    return command", "response": "Parses command strings and returns a Popen - ready list."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef run(command, data=None, timeout=None, kill_timeout=None, env=None, cwd=None):\n\n    command = expand_args(command)\n\n    history = []\n    for c in command:\n\n        if len(history):\n            # due to broken pipe problems pass only first 10 KiB\n            data = history[-1].std_out[0:10*1024]\n\n        cmd = Command(c)\n        try:\n            out, err = cmd.run(data, timeout, kill_timeout, env, cwd)\n            status_code = cmd.returncode\n        except OSError as e:\n            out, err = '', u\"\\n\".join([e.strerror, traceback.format_exc()])\n            status_code = 127\n\n        r = Response(process=cmd)\n\n        r.command = c\n        r.std_out = out\n        r.std_err = err\n        r.status_code = status_code\n\n        history.append(r)\n\n    r = history.pop()\n    r.history = history\n\n    return r", "response": "Executes a given commmand and returns Response."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nspawns a new process from the given command.", "response": "def connect(command, data=None, env=None, cwd=None):\n    \"\"\"Spawns a new process from the given command.\"\"\"\n\n    # TODO: support piped commands\n    command_str = expand_args(command).pop()\n    environ = dict(os.environ)\n    environ.update(env or {})\n\n    process = subprocess.Popen(command_str,\n        universal_newlines=True,\n        shell=False,\n        env=environ,\n        stdin=subprocess.PIPE,\n        stdout=subprocess.PIPE,\n        stderr=subprocess.PIPE,\n        bufsize=0,\n        cwd=cwd,\n    )\n\n    return ConnectedCommand(process=process)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsends a line to the standard input.", "response": "def send(self, str, end='\\n'):\n        \"\"\"Sends a line to std_in.\"\"\"\n        return self._process.stdin.write(str+end)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef bracket_split(source, brackets=('()', '{}', '[]'), strip=False):\n    starts = [e[0] for e in brackets]\n    in_bracket = 0\n    n = 0\n    last = 0\n    while n < len(source):\n        e = source[n]\n        if not in_bracket and e in starts:\n            in_bracket = 1\n            start = n\n            b_start, b_end = brackets[starts.index(e)]\n        elif in_bracket:\n            if e == b_start:\n                in_bracket += 1\n            elif e == b_end:\n                in_bracket -= 1\n                if not in_bracket:\n                    if source[last:start]:\n                        yield source[last:start]\n                    last = n + 1\n                    yield source[start + strip:n + 1 - strip]\n        n += 1\n    if source[last:]:\n        yield source[last:]", "response": "split a string into several lists of strings"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts Py type to PyJs type.", "response": "def Js(val, Clamped=False):\n    '''Converts Py type to PyJs type'''\n    if isinstance(val, PyJs):\n        return val\n    elif val is None:\n        return undefined\n    elif isinstance(val, basestring):\n        return PyJsString(val, StringPrototype)\n    elif isinstance(val, bool):\n        return true if val else false\n    elif isinstance(val, float) or isinstance(val, int) or isinstance(\n            val, long) or (NUMPY_AVAILABLE and isinstance(\n                val,\n                (numpy.int8, numpy.uint8, numpy.int16, numpy.uint16,\n                 numpy.int32, numpy.uint32, numpy.float32, numpy.float64))):\n        # This is supposed to speed things up. may not be the case\n        if val in NUM_BANK:\n            return NUM_BANK[val]\n        return PyJsNumber(float(val), NumberPrototype)\n    elif isinstance(val, FunctionType):\n        return PyJsFunction(val, FunctionPrototype)\n    #elif isinstance(val, ModuleType):\n    #    mod = {}\n    #    for name in dir(val):\n    #        value = getattr(val, name)\n    #        if isinstance(value, ModuleType):\n    #            continue  # prevent recursive module conversion\n    #        try:\n    #            jsval = HJs(value)\n    #        except RuntimeError:\n    #            print 'Could not convert %s to PyJs object!' % name\n    #            continue\n    #        mod[name] = jsval\n    #    return Js(mod)\n    #elif isintance(val, ClassType):\n\n    elif isinstance(val, dict):  # convert to object\n        temp = PyJsObject({}, ObjectPrototype)\n        for k, v in six.iteritems(val):\n            temp.put(Js(k), Js(v))\n        return temp\n    elif isinstance(val, (list, tuple)):  #Convert to array\n        return PyJsArray(val, ArrayPrototype)\n    # convert to typedarray\n    elif isinstance(val, JsObjectWrapper):\n        return val.__dict__['_obj']\n    elif NUMPY_AVAILABLE and isinstance(val, numpy.ndarray):\n        if val.dtype == numpy.int8:\n            return PyJsInt8Array(val, Int8ArrayPrototype)\n        elif val.dtype == numpy.uint8 and not Clamped:\n            return PyJsUint8Array(val, Uint8ArrayPrototype)\n        elif val.dtype == numpy.uint8 and Clamped:\n            return PyJsUint8ClampedArray(val, Uint8ClampedArrayPrototype)\n        elif val.dtype == numpy.int16:\n            return PyJsInt16Array(val, Int16ArrayPrototype)\n        elif val.dtype == numpy.uint16:\n            return PyJsUint16Array(val, Uint16ArrayPrototype)\n\n        elif val.dtype == numpy.int32:\n            return PyJsInt32Array(val, Int32ArrayPrototype)\n        elif val.dtype == numpy.uint32:\n            return PyJsUint16Array(val, Uint32ArrayPrototype)\n\n        elif val.dtype == numpy.float32:\n            return PyJsFloat32Array(val, Float32ArrayPrototype)\n        elif val.dtype == numpy.float64:\n            return PyJsFloat64Array(val, Float64ArrayPrototype)\n    else:  # try to convert to js object\n        return py_wrap(val)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef PyJsStrictEq(a, b):\n    '''a===b'''\n    tx, ty = Type(a), Type(b)\n    if tx != ty:\n        return false\n    if tx == 'Undefined' or tx == 'Null':\n        return true\n    if a.is_primitive():  #string bool and number case\n        return Js(a.value == b.value)\n    if a.Class == b.Class == 'PyObjectWrapper':\n        return Js(a.obj == b.obj)\n    return Js(a is b)", "response": "PyJsStrictEq returns a boolean if a == b otherwise false."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef put(self, prop, val, op=None):  #external use!\n        '''Just like in js: self.prop op= val\n           for example when op is '+' it will be self.prop+=val\n           op can be either None for simple assignment or one of:\n           * / % + - << >> & ^ |'''\n        if self.Class == 'Undefined' or self.Class == 'Null':\n            raise MakeError('TypeError',\n                            'Undefined and null dont have properties!')\n        if not isinstance(prop, basestring):\n            prop = prop.to_string().value\n        if NUMPY_AVAILABLE and prop.isdigit():\n            if self.Class == 'Int8Array':\n                val = Js(numpy.int8(val.to_number().value))\n            elif self.Class == 'Uint8Array':\n                val = Js(numpy.uint8(val.to_number().value))\n            elif self.Class == 'Uint8ClampedArray':\n                if val < Js(numpy.uint8(0)):\n                    val = Js(numpy.uint8(0))\n                elif val > Js(numpy.uint8(255)):\n                    val = Js(numpy.uint8(255))\n                else:\n                    val = Js(numpy.uint8(val.to_number().value))\n            elif self.Class == 'Int16Array':\n                val = Js(numpy.int16(val.to_number().value))\n            elif self.Class == 'Uint16Array':\n                val = Js(numpy.uint16(val.to_number().value))\n            elif self.Class == 'Int32Array':\n                val = Js(numpy.int32(val.to_number().value))\n            elif self.Class == 'Uint32Array':\n                val = Js(numpy.uint32(val.to_number().value))\n            elif self.Class == 'Float32Array':\n                val = Js(numpy.float32(val.to_number().value))\n            elif self.Class == 'Float64Array':\n                val = Js(numpy.float64(val.to_number().value))\n            if isinstance(self.buff, numpy.ndarray):\n                self.buff[int(prop)] = int(val.to_number().value)\n        #we need to set the value to the incremented one\n        if op is not None:\n            val = getattr(self.get(prop), OP_METHODS[op])(val)\n        if not self.can_put(prop):\n            return val\n        own_desc = self.get_own_property(prop)\n        if is_data_descriptor(own_desc):\n            if self.Class in [\n                    'Array', 'Int8Array', 'Uint8Array', 'Uint8ClampedArray',\n                    'Int16Array', 'Uint16Array', 'Int32Array', 'Uint32Array',\n                    'Float32Array', 'Float64Array'\n            ]:\n                self.define_own_property(prop, {'value': val})\n            else:\n                self.own[prop]['value'] = val\n            return val\n        desc = self.get_property(prop)\n        if is_accessor_descriptor(desc):\n            desc['set'].call(self, (val, ))\n        else:\n            new = {\n                'value': val,\n                'writable': True,\n                'configurable': True,\n                'enumerable': True\n            }\n            if self.Class in [\n                    'Array', 'Int8Array', 'Uint8Array', 'Uint8ClampedArray',\n                    'Int16Array', 'Uint16Array', 'Int32Array', 'Uint32Array',\n                    'Float32Array', 'Float64Array'\n            ]:\n                self.define_own_property(prop, new)\n            else:\n                self.own[prop] = new\n        return val", "response": "This method will put a property into the object."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the result of JS == compare. result is PyJs type: bool", "response": "def abstract_equality_comparison(self, other):\n        ''' returns the result of JS == compare.\n           result is PyJs type: bool'''\n        tx, ty = self.TYPE, other.TYPE\n        if tx == ty:\n            if tx == 'Undefined' or tx == 'Null':\n                return true\n            if tx == 'Number' or tx == 'String' or tx == 'Boolean':\n                return Js(self.value == other.value)\n            return Js(self is other)  # Object\n        elif (tx == 'Undefined' and ty == 'Null') or (ty == 'Undefined'\n                                                      and tx == 'Null'):\n            return true\n        elif tx == 'Number' and ty == 'String':\n            return self.abstract_equality_comparison(other.to_number())\n        elif tx == 'String' and ty == 'Number':\n            return self.to_number().abstract_equality_comparison(other)\n        elif tx == 'Boolean':\n            return self.to_number().abstract_equality_comparison(other)\n        elif ty == 'Boolean':\n            return self.abstract_equality_comparison(other.to_number())\n        elif (tx == 'String' or tx == 'Number') and other.is_object():\n            return self.abstract_equality_comparison(other.to_primitive())\n        elif (ty == 'String' or ty == 'Number') and self.is_object():\n            return self.to_primitive().abstract_equality_comparison(other)\n        else:\n            return false"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncall a property prop as a method.", "response": "def callprop(self, prop, *args):\n        '''Call a property prop as a method (this will be self).\n\n        NOTE: dont pass this and arguments here, these will be added\n        automatically!'''\n        if not isinstance(prop, basestring):\n            prop = prop.to_string().value\n        cand = self.get(prop)\n        if not cand.is_callable():\n            raise MakeError('TypeError',\n                            '%s is not a function' % cand.typeof())\n        return cand.call(self, args)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _set_name(self, name):\n        '''name is py type'''\n        if self.own.get('name'):\n            self.func_name = name\n            self.own['name']['value'] = Js(name)", "response": "name is py type"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ncalls this function and returns a result", "response": "def call(self, this, args=()):\n        '''Calls this function and returns a result\n        (converted to PyJs type so func can return python types)\n\n        this must be a PyJs object and args must be a python tuple of PyJs objects.\n\n        arguments object is passed automatically and will be equal to Js(args)\n        (tuple converted to arguments object).You dont need to worry about number\n        of arguments you provide if you supply less then missing ones will be set\n        to undefined (but not present in arguments object).\n        And if you supply too much then excess will not be passed\n        (but they will be present in arguments object).\n        '''\n        if not hasattr(args, '__iter__'):  #get rid of it later\n            args = (args, )\n        args = tuple(Js(e) for e in args)  # this wont be needed later\n\n        arguments = PyJsArguments(\n            args, self)  # tuple will be converted to arguments object.\n        arglen = self.argcount  #function expects this number of args.\n        if len(args) > arglen:\n            args = args[0:arglen]\n        elif len(args) < arglen:\n            args += (undefined, ) * (arglen - len(args))\n        args += this, arguments  #append extra params to the arg list\n        try:\n            return Js(self.code(*args))\n        except NotImplementedError:\n            raise\n        except RuntimeError as e:  # maximum recursion\n            raise MakeError(\n                'RangeError', e.message if\n                not isinstance(e, NotImplementedError) else 'Not implemented!')"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nremoves functions and returns new source and 2 dicts.", "response": "def remove_functions(source, all_inline=False):\n    \"\"\"removes functions and returns new source, and 2 dicts.\n        first dict with removed hoisted(global) functions and second with replaced inline functions\"\"\"\n    global INLINE_COUNT\n    inline = {}\n    hoisted = {}\n    n = 0\n    limit = len(source) - 9  # 8 is length of 'function'\n    res = ''\n    last = 0\n    while n < limit:\n        if n and source[n - 1] in IDENTIFIER_PART:\n            n += 1\n            continue\n        if source[n:n + 8] == 'function' and source[n +\n                                                    8] not in IDENTIFIER_PART:\n            if source[:n].rstrip().endswith(\n                    '.'):  # allow function as a property name :)\n                n += 1\n                continue\n            if source[n + 8:].lstrip().startswith(\n                    ':'):  # allow functions inside objects...\n                n += 1\n                continue\n            entered = n\n            res += source[last:n]\n            name = ''\n            n = pass_white(source, n + 8)\n            if source[n] in IDENTIFIER_START:  # hoisted function\n                name, n = parse_identifier(source, n)\n            args, n = pass_bracket(source, n, '()')\n            if not args:\n                raise SyntaxError('Function misses bracket with argnames ()')\n            args = args.strip('() \\n')\n            args = tuple(parse_identifier(e, 0)[0]\n                         for e in argsplit(args)) if args else ()\n            if len(args) - len(set(args)):\n                # I know its legal in JS but python does not allow duplicate argnames\n                # I will not work around it\n                raise SyntaxError(\n                    'Function has duplicate argument names. Its not legal in this implementation. Sorry.'\n                )\n            block, n = pass_bracket(source, n, '{}')\n            if not block:\n                raise SyntaxError(\n                    'Function does not have any code block to execute')\n            mixed = False  # named function expression flag\n            if name and not all_inline:\n                # Here I will distinguish between named function expression (mixed) and a function statement\n                before = source[:entered].rstrip()\n                if any(endswith_keyword(before, e) for e in PRE_EXP_STARTS):\n                    #print 'Ended ith keyword'\n                    mixed = True\n                elif before and before[-1] not in PRE_ALLOWED and not before[\n                        -2:] in INCREMENTS:\n                    #print 'Ended with'+repr(before[-1]), before[-1]=='}'\n                    mixed = True\n                else:\n                    #print 'FUNCTION STATEMENT'\n                    #its a function statement.\n                    # todo remove fucking label if present!\n                    hoisted[name] = block, args\n            if not name or mixed or all_inline:  # its a function expression (can be both named and not named)\n                #print 'FUNCTION EXPRESSION'\n                INLINE_COUNT += 1\n                iname = INLINE_NAME % INLINE_COUNT  # inline name\n                res += ' ' + iname\n                inline['%s@%s' % (\n                    iname, name\n                )] = block, args  #here added real name at the end because it has to be added to the func scope\n            last = n\n        else:\n            n += 1\n    res += source[last:]\n    return res, hoisted, inline"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef ConstructArray(self, py_arr):\n        ''' note py_arr elems are NOT converted to PyJs types!'''\n        arr = self.NewArray(len(py_arr))\n        arr._init(py_arr)\n        return arr", "response": "constructs a new array from a Python array"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ConstructObject(self, py_obj):\n        ''' note py_obj items are NOT converted to PyJs types! '''\n        obj = self.NewObject()\n        for k, v in py_obj.items():\n            obj.put(unicode(k), v)\n        return obj", "response": "constructs a new object from a dictionary"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nadd an op_code with specified args to the tape.", "response": "def emit(self, op_code, *args):\n        ''' Adds op_code with specified args to tape '''\n        self.tape.append(OP_CODES[op_code](*args))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef compile(self, start_loc=0):\n        ''' Records locations of labels and compiles the code '''\n        self.label_locs = {} if self.label_locs is None else self.label_locs\n        loc = start_loc\n        while loc < len(self.tape):\n            if type(self.tape[loc]) == LABEL:\n                self.label_locs[self.tape[loc].num] = loc\n                del self.tape[loc]\n                continue\n            loc += 1\n        self.compiled = True", "response": "Records locations of labels and compiles the code."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncalling a bytecode function func", "response": "def _call(self, func, this, args):\n        ''' Calls a bytecode function func\n            NOTE:  use !ONLY! when calling functions from native methods! '''\n        assert not func.is_native\n        # fake call - the the runner to return to the end of the file\n        old_contexts = self.contexts\n        old_return_locs = self.return_locs\n        old_curr_ctx = self.current_ctx\n\n        self.contexts = [FakeCtx()]\n        self.return_locs = [len(self.tape)]  # target line after return\n\n        # prepare my ctx\n        my_ctx = func._generate_my_context(this, args)\n        self.current_ctx = my_ctx\n\n        # execute dunction\n        ret = self.run(my_ctx, starting_loc=self.label_locs[func.code])\n\n        # bring back old execution\n        self.current_ctx = old_curr_ctx\n        self.contexts = old_contexts\n        self.return_locs = old_return_locs\n\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef pad(num, n=2, sign=False):\n    '''returns n digit string representation of the num'''\n    s = unicode(abs(num))\n    if len(s) < n:\n        s = '0' * (n - len(s)) + s\n    if not sign:\n        return s\n    if num >= 0:\n        return '+' + s\n    else:\n        return '-' + s", "response": "returns n digit string representation of the num"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef replacement_template(rep, source, span, npar):\n    n = 0\n    res = ''\n    while n < len(rep) - 1:\n        char = rep[n]\n        if char == '$':\n            if rep[n + 1] == '$':\n                res += '$'\n                n += 2\n                continue\n            elif rep[n + 1] == '`':\n                # replace with string that is BEFORE match\n                res += source[:span[0]]\n                n += 2\n                continue\n            elif rep[n + 1] == '\\'':\n                # replace with string that is AFTER match\n                res += source[span[1]:]\n                n += 2\n                continue\n            elif rep[n + 1] in DIGS:\n                dig = rep[n + 1]\n                if n + 2 < len(rep) and rep[n + 2] in DIGS:\n                    dig += rep[n + 2]\n                num = int(dig)\n                # we will not do any replacements if we dont have this npar or dig is 0\n                if not num or num > len(npar):\n                    res += '$' + dig\n                else:\n                    # None - undefined has to be replaced with ''\n                    res += npar[num - 1] if npar[num - 1] else ''\n                n += 1 + len(dig)\n                continue\n        res += char\n        n += 1\n    if n < len(rep):\n        res += rep[-1]\n    return res", "response": "Takes the replacement template and some info about the match and returns filled template\n      "}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef fix_js_args(func):\n    '''Use this function when unsure whether func takes this and arguments as its last 2 args.\n       It will append 2 args if it does not.'''\n    fcode = six.get_function_code(func)\n    fargs = fcode.co_varnames[fcode.co_argcount - 2:fcode.co_argcount]\n    if fargs == ('this', 'arguments') or fargs == ('arguments', 'var'):\n        return func\n    code = append_arguments(six.get_function_code(func), ('this', 'arguments'))\n\n    return types.FunctionType(\n        code,\n        six.get_function_globals(func),\n        func.__name__,\n        closure=six.get_function_closure(func))", "response": "Use this function when unsure whether func takes this and arguments as its last 2 args. It will append 2 args if it does not."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef emit(self, what, *args):\n        ''' what can be either name of the op, or node, or a list of statements.'''\n        if isinstance(what, basestring):\n            return self.exe.emit(what, *args)\n        elif isinstance(what, list):\n            self._emit_statement_list(what)\n        else:\n            return getattr(self, what['type'])(**what)", "response": "emit what to the exe and return the result."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn string representation of this object", "response": "def to_key(literal_or_identifier):\n    ''' returns string representation of this object'''\n    if literal_or_identifier['type'] == 'Identifier':\n        return literal_or_identifier['name']\n    elif literal_or_identifier['type'] == 'Literal':\n        k = literal_or_identifier['value']\n        if isinstance(k, float):\n            return unicode(float_repr(k))\n        elif 'regex' in literal_or_identifier:\n            return compose_regex(k)\n        elif isinstance(k, bool):\n            return 'true' if k else 'false'\n        elif k is None:\n            return 'null'\n        else:\n            return unicode(k)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef trans(ele, standard=False):\n    try:\n        node = globals().get(ele['type'])\n        if not node:\n            raise NotImplementedError('%s is not supported!' % ele['type'])\n        if standard:\n            node = node.__dict__[\n                'standard'] if 'standard' in node.__dict__ else node\n        return node(**ele)\n    except:\n        #print ele\n        raise", "response": "Translates esprima syntax tree to python by delegating to appropriate translating node"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef limited(func):\n    '''Decorator limiting resulting line length in order to avoid python parser stack overflow -\n      If expression longer than LINE_LEN_LIMIT characters then it will be moved to upper line\n     USE ONLY ON EXPRESSIONS!!! '''\n\n    def f(standard=False, **args):\n        insert_pos = len(\n            inline_stack.names\n        )  # in case line is longer than limit we will have to insert the lval at current position\n        # this is because calling func will change inline_stack.\n        # we cant use inline_stack.require here because we dont know whether line overflows yet\n        res = func(**args)\n        if len(res) > LINE_LEN_LIMIT:\n            name = inline_stack.require('LONG')\n            inline_stack.names.pop()\n            inline_stack.names.insert(insert_pos, name)\n            res = 'def %s(var=var):\\n    return %s\\n' % (name, res)\n            inline_stack.define(name, res)\n            return name + '()'\n        else:\n            return res\n\n    f.__dict__['standard'] = func\n    return f", "response": "Decorator to limit resulting line length in order to avoid python parser stack overflow -\n      If expression longer than LINE_LEN_LIMIT characters then it will be moved to upper line\n     USE ONLY ON EXPRESSIONS!!!"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndoing not chceck whether t is not resticted or internal", "response": "def is_lval(t):\n    \"\"\"Does not chceck whether t is not resticted or internal\"\"\"\n    if not t:\n        return False\n    i = iter(t)\n    if i.next() not in IDENTIFIER_START:\n        return False\n    return all(e in IDENTIFIER_PART for e in i)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef is_valid_lval(t):\n    if not is_internal(t) and is_lval(t) and t not in RESERVED_NAMES:\n        return True\n    return False", "response": "Checks whether t is a valid JS identifier name."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ntranslate a javascript source code to esprima syntax tree.", "response": "def translate_js(js, HEADER=DEFAULT_HEADER, use_compilation_plan=False):\n    \"\"\"js has to be a javascript source code.\n       returns equivalent python code.\"\"\"\n    if use_compilation_plan and not '//' in js and not '/*' in js:\n        return translate_js_with_compilation_plan(js, HEADER=HEADER)\n    parser = pyjsparser.PyJsParser()\n    parsed = parser.parse(js)  # js to esprima syntax tree\n    # Another way of doing that would be with my auto esprima translation but its much slower and causes import problems:\n    # parsed = esprima.parse(js).to_dict()\n    translating_nodes.clean_stacks()\n    return HEADER + translating_nodes.trans(\n        parsed)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef translate_js_with_compilation_plan(js, HEADER=DEFAULT_HEADER):\n\n    match_increaser_str, match_increaser_num, compilation_plan = get_compilation_plan(\n        js)\n\n    cp_hash = hashlib.md5(compilation_plan.encode('utf-8')).digest()\n    try:\n        python_code = cache[cp_hash]['proto_python_code']\n    except:\n        parser = pyjsparser.PyJsParser()\n        parsed = parser.parse(compilation_plan)  # js to esprima syntax tree\n        # Another way of doing that would be with my auto esprima translation but its much slower and causes import problems:\n        # parsed = esprima.parse(js).to_dict()\n        translating_nodes.clean_stacks()\n        python_code = translating_nodes.trans(\n            parsed)  # syntax tree to python code\n        cache[cp_hash] = {\n            'compilation_plan': compilation_plan,\n            'proto_python_code': python_code,\n        }\n\n    python_code = match_increaser_str.wrap_up(python_code)\n    python_code = match_increaser_num.wrap_up(python_code)\n\n    return HEADER + python_code", "response": "Translate a javascript source code with a compilation plan."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nimporting from javascript source file.", "response": "def import_js(path, lib_name, globals):\n    \"\"\"Imports from javascript source file.\n      globals is your globals()\"\"\"\n    with codecs.open(path_as_local(path), \"r\", \"utf-8\") as f:\n        js = f.read()\n    e = EvalJs()\n    e.execute(js)\n    var = e.context['var']\n    globals[lib_name] = var.to_python()"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntranslates a JS file to python and saves it to the output path.", "response": "def translate_file(input_path, output_path):\n    '''\n    Translates input JS file to python and saves the it to the output path.\n    It appends some convenience code at the end so that it is easy to import JS objects.\n\n    For example we have a file 'example.js' with:   var a = function(x) {return x}\n    translate_file('example.js', 'example.py')\n\n    Now example.py can be easily importend and used:\n    >>> from example import example\n    >>> example.a(30)\n    30\n    '''\n    js = get_file_contents(input_path)\n\n    py_code = translate_js(js)\n    lib_name = os.path.basename(output_path).split('.')[0]\n    head = '__all__ = [%s]\\n\\n# Don\\'t look below, you will not understand this Python code :) I don\\'t.\\n\\n' % repr(\n        lib_name)\n    tail = '\\n\\n# Add lib to the module scope\\n%s = var.to_python()' % lib_name\n    out = head + py_code + tail\n    write_file_contents(output_path, out)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run_file(path_or_file, context=None):\n    ''' Context must be EvalJS object. Runs given path as a JS program. Returns (eval_value, context).\n    '''\n    if context is None:\n        context = EvalJs()\n    if not isinstance(context, EvalJs):\n        raise TypeError('context must be the instance of EvalJs')\n    eval_value = context.eval(get_file_contents(path_or_file))\n    return eval_value, context", "response": "Runs given path as a JS program. Returns the eval_value and context."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nexecute javascript js in current context", "response": "def execute(self, js=None, use_compilation_plan=False):\n        \"\"\"executes javascript js in current context\n\n        During initial execute() the converted js is cached for re-use. That means next time you\n        run the same javascript snippet you save many instructions needed to parse and convert the\n        js code to python code.\n\n        This cache causes minor overhead (a cache dicts is updated) but the Js=>Py conversion process\n        is typically expensive compared to actually running the generated python code.\n\n        Note that the cache is just a dict, it has no expiration or cleanup so when running this\n        in automated situations with vast amounts of snippets it might increase memory usage.\n        \"\"\"\n        try:\n            cache = self.__dict__['cache']\n        except KeyError:\n            cache = self.__dict__['cache'] = {}\n        hashkey = hashlib.md5(js.encode('utf-8')).digest()\n        try:\n            compiled = cache[hashkey]\n        except KeyError:\n            code = translate_js(\n                js, '', use_compilation_plan=use_compilation_plan)\n            compiled = cache[hashkey] = compile(code, '<EvalJS snippet>',\n                                                'exec')\n        exec (compiled, self._context)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nevaluate expression in current context and returns its value", "response": "def eval(self, expression, use_compilation_plan=False):\n        \"\"\"evaluates expression in current context and returns its value\"\"\"\n        code = 'PyJsEvalResult = eval(%s)' % json.dumps(expression)\n        self.execute(code, use_compilation_plan=use_compilation_plan)\n        return self['PyJsEvalResult']"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef execute_debug(self, js):\n        code = translate_js(js, '')\n        # make sure you have a temp folder:\n        filename = 'temp' + os.sep + '_' + hashlib.md5(\n            code.encode(\"utf-8\")).hexdigest() + '.py'\n        try:\n            with open(filename, mode='w') as f:\n                f.write(code)\n            with open(filename, \"r\") as f:\n                pyCode = compile(f.read(), filename, 'exec')\n                exec(pyCode, self._context)\n                \n        except Exception as err:\n            raise err\n        finally:\n            os.remove(filename)\n            try:\n                os.remove(filename + 'c')\n            except:\n                pass", "response": "execute the javascript js in current context\n       "}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef eval_debug(self, expression):\n        code = 'PyJsEvalResult = eval(%s)' % json.dumps(expression)\n        self.execute_debug(code)\n        return self['PyJsEvalResult']", "response": "evaluates expression in current context and returns its value\n        as opposed to the execute method"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef console(self):\n        while True:\n            if six.PY2:\n                code = raw_input('>>> ')\n            else:\n                code = input('>>>')\n            try:\n                print(self.eval(code))\n            except KeyboardInterrupt:\n                break\n            except Exception as e:\n                import traceback\n                if DEBUG:\n                    sys.stderr.write(traceback.format_exc())\n                else:\n                    sys.stderr.write('EXCEPTION: ' + str(e) + '\\n')\n                time.sleep(0.01)", "response": "starts to interact with interactive console"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef to_key(literal_or_identifier):\n    ''' returns string representation of this object'''\n    if literal_or_identifier['type'] == 'Identifier':\n        return literal_or_identifier['name']\n    elif literal_or_identifier['type'] == 'Literal':\n        k = literal_or_identifier['value']\n        if isinstance(k, float):\n            return unicode(float_repr(k))\n        elif 'regex' in literal_or_identifier:\n            return compose_regex(k)\n        elif isinstance(k, bool):\n            return u'true' if k else u'false'\n        elif k is None:\n            return u'null'\n        else:\n            return unicode(k)", "response": "returns string representation of this object"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ntranslates functions and all nested functions to Python code.", "response": "def translate_func(name, block, args):\n    \"\"\"Translates functions and all nested functions to Python code.\n       name -  name of that function (global functions will be available under var while\n            inline will be available directly under this name )\n       block - code of the function (*with* brackets {} )\n       args - arguments that this function takes\"\"\"\n    inline = name.startswith('PyJsLvalInline')\n    real_name = ''\n    if inline:\n        name, real_name = name.split('@')\n    arglist = ', '.join(args) + ', ' if args else ''\n    code = '@Js\\ndef %s(%sthis, arguments, var=var):\\n' % (name, arglist)\n    # register local variables\n    scope = \"'this':this, 'arguments':arguments\"  #it will be a simple dictionary\n    for arg in args:\n        scope += ', %s:%s' % (repr(arg), arg)\n    if real_name:\n        scope += ', %s:%s' % (repr(real_name), name)\n    code += indent('var = Scope({%s}, var)\\n' % scope)\n    block, nested_hoisted, nested_inline = remove_functions(block)\n    py_code, to_register = translate_flow(block)\n    #register variables declared with var and names of hoisted functions.\n    to_register += nested_hoisted.keys()\n    if to_register:\n        code += indent('var.registers(%s)\\n' % str(to_register))\n    for nested_name, info in nested_hoisted.iteritems():\n        nested_block, nested_args = info\n        new_code = translate_func('PyJsLvalTempHoisted', nested_block,\n                                  nested_args)\n        # Now put definition of hoisted function on the top\n        code += indent(new_code)\n        code += indent(\n            'PyJsLvalTempHoisted.func_name = %s\\n' % repr(nested_name))\n        code += indent(\n            'var.put(%s, PyJsLvalTempHoisted)\\n' % repr(nested_name))\n    for nested_name, info in nested_inline.iteritems():\n        nested_block, nested_args = info\n        new_code = translate_func(nested_name, nested_block, nested_args)\n        # Inject definitions of inline functions just before usage\n        # nested inline names have this format : LVAL_NAME@REAL_NAME\n        py_code = inject_before_lval(py_code,\n                                     nested_name.split('@')[0], new_code)\n    if py_code.strip():\n        code += indent(py_code)\n    return code"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef pass_bracket(source, start, bracket='()'):\n    e = bracket_split(source[start:], [bracket], False)\n    try:\n        cand = e.next()\n    except StopIteration:\n        return None, None\n    if not cand.strip():  #white space...\n        try:\n            res = e.next()\n            return res, start + len(cand) + len(res)\n        except StopIteration:\n            return None, None\n    elif cand[-1] == bracket[1]:\n        return cand, start + len(cand)\n    else:\n        return None, None", "response": "Returns content of brackets with brackets and first pos after brackets\n     Otherwise None"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef except_token(source, start, token, throw=True):\n    start = pass_white(source, start)\n    if start < len(source) and source[start] == token:\n        return start + 1\n    if throw:\n        raise SyntaxError('Missing token. Expected %s' % token)\n    return None", "response": "Returns the position after token if found."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the position after the keyword if found else None.", "response": "def except_keyword(source, start, keyword):\n    \"\"\" Returns position after keyword if found else None\n        Note: skips white space\"\"\"\n    start = pass_white(source, start)\n    kl = len(keyword)  #keyword len\n    if kl + start > len(source):\n        return None\n    if source[start:start + kl] != keyword:\n        return None\n    if kl + start < len(source) and source[start + kl] in IDENTIFIER_PART:\n        return None\n    return start + kl"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nparse a string from start and returns first identifier if identifier invalid and raises SyntaxError otherwise returns None", "response": "def parse_identifier(source, start, throw=True):\n    \"\"\"passes white space from start and returns first identifier,\n       if identifier invalid and throw raises SyntaxError otherwise returns None\"\"\"\n    start = pass_white(source, start)\n    end = start\n    if not end < len(source):\n        if throw:\n            raise SyntaxError('Missing identifier!')\n        return None\n    if source[end] not in IDENTIFIER_START:\n        if throw:\n            raise SyntaxError('Invalid identifier start: \"%s\"' % source[end])\n        return None\n    end += 1\n    while end < len(source) and source[end] in IDENTIFIER_PART:\n        end += 1\n    if not is_valid_lval(source[start:end]):\n        if throw:\n            raise SyntaxError(\n                'Invalid identifier name: \"%s\"' % source[start:end])\n        return None\n    return source[start:end], end"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef argsplit(args, sep=','):\n    parsed_len = 0\n    last = 0\n    splits = []\n    for e in bracket_split(args, brackets=['()', '[]', '{}']):\n        if e[0] not in {'(', '[', '{'}:\n            for i, char in enumerate(e):\n                if char == sep:\n                    splits.append(args[last:parsed_len + i])\n                    last = parsed_len + i + 1\n        parsed_len += len(e)\n    splits.append(args[last:])\n    return splits", "response": "used to split JS args into array and object elements"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nspecialize function splitting text at add and sub operators.", "response": "def split_add_ops(text):\n    \"\"\"Specialized function splitting text at add/sub operators.\n    Operands are *not* translated. Example result ['op1', '+', 'op2', '-', 'op3']\"\"\"\n    n = 0\n    text = text.replace('++', '##').replace(\n        '--', '@@')  #text does not normally contain any of these\n    spotted = False  # set to true if noticed anything other than +- or white space\n    last = 0\n    while n < len(text):\n        e = text[n]\n        if e == '+' or e == '-':\n            if spotted:\n                yield text[last:n].replace('##', '++').replace('@@', '--')\n                yield e\n                last = n + 1\n                spotted = False\n        elif e == '/' or e == '*' or e == '%':\n            spotted = False\n        elif e != ' ':\n            spotted = True\n        n += 1\n    yield text[last:n].replace('##', '++').replace('@@', '--')"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsplits at any of the specified set of items in the specified order", "response": "def split_at_any(text,\n                 lis,\n                 translate=False,\n                 not_before=[],\n                 not_after=[],\n                 validitate=None):\n    \"\"\" doc \"\"\"\n    lis.sort(key=lambda x: len(x), reverse=True)\n    last = 0\n    n = 0\n    text_len = len(text)\n    while n < text_len:\n        if any(text[:n].endswith(e)\n               for e in not_before):  #Cant end with end before\n            n += 1\n            continue\n        for e in lis:\n            s = len(e)\n            if s + n > text_len:\n                continue\n            if validitate and not validitate(e, text[:n], text[n + s:]):\n                continue\n            if any(text[n + s:].startswith(e)\n                   for e in not_after):  #Cant end with end before\n                n += 1\n                break\n            if e == text[n:n + s]:\n                yield text[last:n] if not translate else translate(\n                    text[last:n])\n                yield e\n                n += s\n                last = n\n                break\n        else:\n            n += 1\n    yield text[last:n] if not translate else translate(text[last:n])"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nworking like text. split but separated fragments cant end with not_before and start with not_after", "response": "def split_at_single(text, sep, not_before=[], not_after=[]):\n    \"\"\"Works like text.split(sep) but separated fragments\n    cant end with not_before or start with not_after\"\"\"\n    n = 0\n    lt, s = len(text), len(sep)\n    last = 0\n    while n < lt:\n        if not s + n > lt:\n            if sep == text[n:n + s]:\n                if any(text[last:n].endswith(e) for e in not_before):\n                    pass\n                elif any(text[n + s:].startswith(e) for e in not_after):\n                    pass\n                else:\n                    yield text[last:n]\n                    last = n + s\n                    n += s - 1\n        n += 1\n    yield text[last:]"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns Python array from Js array", "response": "def to_arr(this):\n    \"\"\"Returns Python array from Js array\"\"\"\n    return [this.get(str(e)) for e in xrange(len(this))]"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef transform_crap(code):  #needs some more tests\n    ind = code.rfind('?')\n    if ind == -1:\n        return code\n    sep = code.find(':', ind)\n    if sep == -1:\n        raise SyntaxError('Invalid ?: syntax (probably missing \":\" )')\n    beg = max(code.rfind(':', 0, ind), code.find('?', 0, ind)) + 1\n    end = code.find(':', sep + 1)\n    end = len(code) if end == -1 else end\n    formula = '(' + code[ind + 1:sep] + ' if ' + code[\n        beg:ind] + ' else ' + code[sep + 1:end] + ')'\n    return transform_crap(code[:beg] + formula + code[end:])", "response": "Transforms this? : crap into if else python syntax"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nperforms this operation on a list from right to left *", "response": "def rl(self, lis, op):\n        \"\"\"performs this operation on a list from *right to left*\n        op must take 2 args\n        a,b,c  => op(a, op(b, c))\"\"\"\n        it = reversed(lis)\n        res = trans(it.next())\n        for e in it:\n            e = trans(e)\n            res = op(e, res)\n        return res"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef lr(self, lis, op):\n        it = iter(lis)\n        res = trans(it.next())\n        for e in it:\n            e = trans(e)\n            res = op(res, e)\n        return res", "response": "performs this operation on a list from left to right *"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef translate(self):\n        if not self.code:\n            return ''\n        new = bracket_replace(self.code)\n        #Check comma operator:\n        cand = new.split(',')  #every comma in new must be an operator\n        if len(cand) > 1:  #LR\n            return self.lr(cand, js_comma)\n        #Check = operator:\n        # dont split at != or !== or == or === or <= or >=\n        #note <<=, >>= or this >>> will NOT be supported\n        # maybe I will change my mind later\n        # Find this crappy ?:\n        if '?' in new:\n            cond_ind = new.find('?')\n            tenary_start = 0\n            for ass in re.finditer(ASSIGNMENT_MATCH, new):\n                cand = ass.span()[1]\n                if cand < cond_ind:\n                    tenary_start = cand\n                else:\n                    break\n            actual_tenary = new[tenary_start:]\n            spl = ''.join(split_at_any(new, [':', '?'], translate=trans))\n            tenary_translation = transform_crap(spl)\n            assignment = new[:tenary_start] + ' PyJsConstantTENARY'\n            return trans(assignment).replace('PyJsConstantTENARY',\n                                             tenary_translation)\n        cand = list(split_at_single(new, '=', ['!', '=', '<', '>'], ['=']))\n        if len(cand) > 1:  # RL\n            it = reversed(cand)\n            res = trans(it.next())\n            for e in it:\n                e = e.strip()\n                if not e:\n                    raise SyntaxError('Missing left-hand in assignment!')\n                op = ''\n                if e[-2:] in OP_METHODS:\n                    op = ',' + e[-2:].__repr__()\n                    e = e[:-2]\n                elif e[-1:] in OP_METHODS:\n                    op = ',' + e[-1].__repr__()\n                    e = e[:-1]\n                e = trans(e)\n                #Now replace last get method with put and change args\n                c = list(bracket_split(e, ['()']))\n                beg, arglist = ''.join(c[:-1]).strip(), c[-1].strip(\n                )  #strips just to make sure... I will remove it later\n                if beg[-4:] != '.get':\n                    raise SyntaxError('Invalid left-hand side in assignment')\n                beg = beg[0:-3] + 'put'\n                arglist = arglist[0:-1] + ', ' + res + op + ')'\n                res = beg + arglist\n            return res\n        #Now check remaining 2 arg operators that are not handled by python\n        #They all have Left to Right (LR) associativity\n        order = [OR, AND, BOR, BXOR, BAND, EQS, COMPS, BSHIFTS, ADDS, MULTS]\n        # actually we dont need OR and AND because they can be handled easier. But just for fun\n        dangerous = ['<', '>']\n        for typ in order:\n            #we have to use special method for ADDS since they can be also unary operation +/++ or -/-- FUCK\n            if '+' in typ:\n                cand = list(split_add_ops(new))\n            else:\n                #dont translate. cant start or end on dangerous op.\n                cand = list(\n                    split_at_any(\n                        new,\n                        typ.keys(),\n                        False,\n                        dangerous,\n                        dangerous,\n                        validitate=comb_validitator))\n            if not len(cand) > 1:\n                continue\n            n = 1\n            res = trans(cand[0])\n            if not res:\n                raise SyntaxError(\"Missing operand!\")\n            while n < len(cand):\n                e = cand[n]\n                if not e:\n                    raise SyntaxError(\"Missing operand!\")\n                if n % 2:\n                    op = typ[e]\n                else:\n                    res = op(res, trans(e))\n                n += 1\n            return res\n        #Now replace unary operators - only they are left\n        cand = list(\n            split_at_any(\n                new, UNARY.keys(), False, validitate=unary_validitator))\n        if len(cand) > 1:  #contains unary operators\n            if '++' in cand or '--' in cand:  #it cant contain both ++ and --\n                if '--' in cand:\n                    op = '--'\n                    meths = js_post_dec, js_pre_dec\n                else:\n                    op = '++'\n                    meths = js_post_inc, js_pre_inc\n                pos = cand.index(op)\n                if cand[pos - 1].strip():  # post increment\n                    a = cand[pos - 1]\n                    meth = meths[0]\n                elif cand[pos + 1].strip():  #pre increment\n                    a = cand[pos + 1]\n                    meth = meths[1]\n                else:\n                    raise SyntaxError('Invalid use of ++ operator')\n                if cand[pos + 2:]:\n                    raise SyntaxError('Too many operands')\n                operand = meth(trans(a))\n                cand = cand[:pos - 1]\n            # now last cand should be operand and every other odd element should be empty\n            else:\n                operand = trans(cand[-1])\n                del cand[-1]\n            for i, e in enumerate(reversed(cand)):\n                if i % 2:\n                    if e.strip():\n                        raise SyntaxError('Too many operands')\n                else:\n                    operand = UNARY[e](operand)\n            return operand\n        #Replace brackets\n        if new[0] == '@' or new[0] == '#':\n            if len(\n                    list(bracket_split(new, ('#{', '@}')))\n            ) == 1:  # we have only one bracket, otherwise pseudobracket like @@....\n                assert new in REPL\n                if new[0] == '#':\n                    raise SyntaxError(\n                        '[] cant be used as brackets! Use () instead.')\n                return '(' + trans(REPL[new][1:-1]) + ')'\n        #Replace function calls and prop getters\n        # 'now' must be a reference like: a or b.c.d but it can have also calls or getters ( for example a[\"b\"](3))\n        #From here @@ means a function call and ## means get operation (note they dont have to present)\n        it = bracket_split(new, ('#{', '@}'))\n        res = []\n        for e in it:\n            if e[0] != '#' and e[0] != '@':\n                res += [x.strip() for x in e.split('.')]\n            else:\n                res += [e.strip()]\n        # res[0] can be inside @@ (name)...\n        res = filter(lambda x: x, res)\n        if is_internal(res[0]):\n            out = res[0]\n        elif res[0][0] in {'#', '@'}:\n            out = '(' + trans(REPL[res[0]][1:-1]) + ')'\n        elif is_valid_lval(\n                res[0]) or res[0] in {'this', 'false', 'true', 'null'}:\n            out = 'var.get(' + res[0].__repr__() + ')'\n        else:\n            if is_reserved(res[0]):\n                raise SyntaxError('Unexpected reserved word: \"%s\"' % res[0])\n            raise SyntaxError('Invalid identifier: \"%s\"' % res[0])\n        if len(res) == 1:\n            return out\n        n = 1\n        while n < len(res):  #now every func call is a prop call\n            e = res[n]\n            if e[0] == '@':  # direct call\n                out += trans_args(REPL[e])\n                n += 1\n                continue\n            args = False  #assume not prop call\n            if n + 1 < len(res) and res[n + 1][0] == '@':  #prop call\n                args = trans_args(REPL[res[n + 1]])[1:]\n                if args != ')':\n                    args = ',' + args\n            if e[0] == '#':\n                prop = trans(REPL[e][1:-1])\n            else:\n                if not is_lval(e):\n                    raise SyntaxError('Invalid identifier: \"%s\"' % e)\n                prop = e.__repr__()\n            if args:  # prop call\n                n += 1\n                out += '.callprop(' + prop + args\n            else:  #prop get\n                out += '.get(' + prop + ')'\n            n += 1\n        return out", "response": "Translates outer operation and calls translate on inner operation."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns None if not found other functions that begin with do_", "response": "def do_statement(source, start):\n    \"\"\"returns none if not found other functions that begin with 'do_' raise\n    also this do_ type function passes white space\"\"\"\n    start = pass_white(source, start)\n    # start is the fist position after initial start that is not a white space or \\n\n    if not start < len(source):  #if finished parsing return None\n        return None, start\n    if any(startswith_keyword(source[start:], e) for e in {'case', 'default'}):\n        return None, start\n    rest = source[start:]\n    for key, meth in KEYWORD_METHODS.iteritems(\n    ):  # check for statements that are uniquely defined by their keywords\n        if rest.startswith(key):\n            # has to startwith this keyword and the next letter after keyword must be either EOF or not in IDENTIFIER_PART\n            if len(key) == len(rest) or rest[len(key)] not in IDENTIFIER_PART:\n                return meth(source, start)\n    if rest[0] == '{':  #Block\n        return do_block(source, start)\n    # Now only label and expression left\n    cand = parse_identifier(source, start, False)\n    if cand is not None:  # it can mean that its a label\n        label, cand_start = cand\n        cand_start = pass_white(source, cand_start)\n        if source[cand_start] == ':':\n            return do_label(source, start)\n    return do_expression(source, start)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmatching a string at the given position", "response": "def match(self, string, pos):\n        '''string is of course a py string'''\n        return self.pat.match(string, int(pos))"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nchecks if n is empty object", "response": "def is_empty_object(n, last):\n    \"\"\"n may be the inside of block or object\"\"\"\n    if n.strip():\n        return False\n    # seems to be but can be empty code\n    last = last.strip()\n    markers = {\n        ')',\n        ';',\n    }\n    if not last or last[-1] in markers:\n        return False\n    return True"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef is_object(n, last):\n    if is_empty_object(n, last):\n        return True\n    if not n.strip():\n        return False\n    #Object contains lines of code so it cant be an object\n    if len(argsplit(n, ';')) > 1:\n        return False\n    cands = argsplit(n, ',')\n    if not cands[-1].strip():\n        return True  # {xxxx,} empty after last , it must be an object\n    for cand in cands:\n        cand = cand.strip()\n        # separate each candidate element at : in dict and check whether they are correct...\n        kv = argsplit(cand, ':')\n        if len(\n                kv\n        ) > 2:  # set the len of kv to 2 because of this stupid : expression\n            kv = kv[0], ':'.join(kv[1:])\n\n        if len(kv) == 2:\n            # key value pair, check whether not label or ?:\n            k, v = kv\n            if not is_lval(k.strip()):\n                return False\n            v = v.strip()\n            if v.startswith('function'):\n                continue\n            #will fail on label... {xxx: while {}}\n            if v[0] == '{':  # value cant be a code block\n                return False\n            for e in KEYWORD_METHODS:\n                # if v starts with any statement then return false\n                if v.startswith(e) and len(e) < len(v) and v[len(\n                        e)] not in IDENTIFIER_PART:\n                    return False\n        elif not (cand.startswith('set ') or cand.startswith('get ')):\n            return False\n    return True", "response": "check if n is an object"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef remove_arrays(code, count=1):\n    res = ''\n    last = ''\n    replacements = {}\n    for e in bracket_split(code, ['[]']):\n        if e[0] == '[':\n            if is_array(last):\n                name = ARRAY_LVAL % count\n                res += ' ' + name\n                replacements[name] = e\n                count += 1\n            else:  # pseudo array. But pseudo array can contain true array. for example a[['d'][3]] has 2 pseudo and 1 true array\n                cand, new_replacements, count = remove_arrays(e[1:-1], count)\n                res += '[%s]' % cand\n                replacements.update(new_replacements)\n        else:\n            res += e\n        last = e\n    return res, replacements, count", "response": "removes arrays and replaces them with ARRAY_LVALS\nATTRIBS"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef translate_array(array, lval, obj_count=1, arr_count=1):\n    array = array[1:-1]\n    array, obj_rep, obj_count = remove_objects(array, obj_count)\n    array, arr_rep, arr_count = remove_arrays(array, arr_count)\n    #functions can be also defined in arrays, this caused many problems since in Python\n    # functions cant be defined inside literal\n    # remove functions (they dont contain arrays or objects so can be translated easily)\n    # hoisted functions are treated like inline\n    array, hoisted, inline = functions.remove_functions(array, all_inline=True)\n    assert not hoisted\n    arr = []\n    # separate elements in array\n    for e in argsplit(array, ','):\n        # translate expressions in array PyJsLvalInline will not be translated!\n        e = exp_translator(e.replace('\\n', ''))\n        arr.append(e if e else 'None')\n    arr = '%s = Js([%s])\\n' % (lval, ','.join(arr))\n    #But we can have more code to add to define arrays/objects/functions defined inside this array\n    # translate nested objects:\n    # functions:\n    for nested_name, nested_info in inline.iteritems():\n        nested_block, nested_args = nested_info\n        new_def = FUNC_TRANSLATOR(nested_name, nested_block, nested_args)\n        arr = new_def + arr\n    for lval, obj in obj_rep.iteritems():\n        new_def, obj_count, arr_count = translate_object(\n            obj, lval, obj_count, arr_count)\n        # add object definition BEFORE array definition\n        arr = new_def + arr\n    for lval, obj in arr_rep.iteritems():\n        new_def, obj_count, arr_count = translate_array(\n            obj, lval, obj_count, arr_count)\n        # add object definition BEFORE array definition\n        arr = new_def + arr\n    return arr, obj_count, arr_count", "response": "Translate a js array into a python code that can be used in the Python interpreter."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _ensure_regexp(source, n):  #<- this function has to be improved\n    '''returns True if regexp starts at n else returns False\n      checks whether it is not a division '''\n    markers = '(+~\"\\'=[%:?!*^|&-,;/\\\\'\n    k = 0\n    while True:\n        k += 1\n        if n - k < 0:\n            return True\n        char = source[n - k]\n        if char in markers:\n            return True\n        if char != ' ' and char != '\\n':\n            break\n    return False", "response": "returns True if regexp starts at n else returns False"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef parse_num(source, start, charset):\n    while start < len(source) and source[start] in charset:\n        start += 1\n    return start", "response": "Returns a first index > = start of chat not in charset"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning end of exponential, raises SyntaxError if failed", "response": "def parse_exponent(source, start):\n    \"\"\"returns end of exponential, raises SyntaxError if failed\"\"\"\n    if not source[start] in {'e', 'E'}:\n        if source[start] in IDENTIFIER_PART:\n            raise SyntaxError('Invalid number literal!')\n        return start\n    start += 1\n    if source[start] in {'-', '+'}:\n        start += 1\n    FOUND = False\n    # we need at least one dig after exponent\n    while source[start] in NUMS:\n        FOUND = True\n        start += 1\n    if not FOUND or source[start] in IDENTIFIER_PART:\n        raise SyntaxError('Invalid number literal!')\n    return start"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreplacing Strings and Regexp literals in the source code with their corresponding constant.", "response": "def remove_constants(source):\n    '''Replaces Strings and Regexp literals in the source code with\n       identifiers and *removes comments*. Identifier is of the format:\n\n       PyJsStringConst(String const number)_ - for Strings\n       PyJsRegExpConst(RegExp const number)_ - for RegExps\n\n       Returns dict which relates identifier and replaced constant.\n\n       Removes single line and multiline comments from JavaScript source code\n       Pseudo comments (inside strings) will not be removed.\n\n       For example this line:\n       var x = \"/*PSEUDO COMMENT*/ TEXT //ANOTHER PSEUDO COMMENT\"\n       will be unaltered'''\n    source = ' ' + source + '\\n'\n    comments = []\n    inside_comment, single_comment = False, False\n    inside_single, inside_double = False, False\n    inside_regexp = False\n    regexp_class_count = 0\n    n = 0\n    while n < len(source):\n        char = source[n]\n        if char == '\"' and not (inside_comment or inside_single\n                                or inside_regexp):\n            if not _is_cancelled(source, n):\n                if inside_double:\n                    inside_double[1] = n + 1\n                    comments.append(inside_double)\n                    inside_double = False\n                else:\n                    inside_double = [n, None, 0]\n        elif char == \"'\" and not (inside_comment or inside_double\n                                  or inside_regexp):\n            if not _is_cancelled(source, n):\n                if inside_single:\n                    inside_single[1] = n + 1\n                    comments.append(inside_single)\n                    inside_single = False\n                else:\n                    inside_single = [n, None, 0]\n        elif (inside_single or inside_double):\n            if char in LINE_TERMINATOR:\n                if _is_cancelled(source, n):\n                    if char == CR and source[n + 1] == LF:\n                        n += 1\n                    n += 1\n                    continue\n                else:\n                    raise SyntaxError(\n                        'Invalid string literal. Line terminators must be escaped!'\n                    )\n        else:\n            if inside_comment:\n                if single_comment:\n                    if char in LINE_TERMINATOR:\n                        inside_comment[1] = n\n                        comments.append(inside_comment)\n                        inside_comment = False\n                        single_comment = False\n                else:  # Multiline\n                    if char == '/' and source[n - 1] == '*':\n                        inside_comment[1] = n + 1\n                        comments.append(inside_comment)\n                        inside_comment = False\n            elif inside_regexp:\n                if not quiting_regexp:\n                    if char in LINE_TERMINATOR:\n                        raise SyntaxError(\n                            'Invalid regexp literal. Line terminators cant appear!'\n                        )\n                    if _is_cancelled(source, n):\n                        n += 1\n                        continue\n                    if char == '[':\n                        regexp_class_count += 1\n                    elif char == ']':\n                        regexp_class_count = max(regexp_class_count - 1, 0)\n                    elif char == '/' and not regexp_class_count:\n                        quiting_regexp = True\n                else:\n                    if char not in IDENTIFIER_START:\n                        inside_regexp[1] = n\n                        comments.append(inside_regexp)\n                        inside_regexp = False\n            elif char == '/' and source[n - 1] == '/':\n                single_comment = True\n                inside_comment = [n - 1, None, 1]\n            elif char == '*' and source[n - 1] == '/':\n                inside_comment = [n - 1, None, 1]\n            elif char == '/' and source[n + 1] not in ('/', '*'):\n                if not _ensure_regexp(source, n):  #<- improve this one\n                    n += 1\n                    continue  #Probably just a division\n                quiting_regexp = False\n                inside_regexp = [n, None, 2]\n            elif not (inside_comment or inside_regexp):\n                if (char in NUMS and\n                        source[n - 1] not in IDENTIFIER_PART) or char == '.':\n                    if char == '.':\n                        k = parse_num(source, n + 1, NUMS)\n                        if k == n + 1:  # just a stupid dot...\n                            n += 1\n                            continue\n                        k = parse_exponent(source, k)\n                    elif char == '0' and source[n + 1] in {\n                            'x', 'X'\n                    }:  #Hex number probably\n                        k = parse_num(source, n + 2, HEX)\n                        if k == n + 2 or source[k] in IDENTIFIER_PART:\n                            raise SyntaxError('Invalid hex literal!')\n                    else:  #int or exp or flot or exp flot\n                        k = parse_num(source, n + 1, NUMS)\n                        if source[k] == '.':\n                            k = parse_num(source, k + 1, NUMS)\n                        k = parse_exponent(source, k)\n                    comments.append((n, k, 3))\n                    n = k\n                    continue\n        n += 1\n    res = ''\n    start = 0\n    count = 0\n    constants = {}\n    for end, next_start, typ in comments:\n        res += source[start:end]\n        start = next_start\n        if typ == 0:  # String\n            name = StringName\n        elif typ == 1:  # comment\n            continue\n        elif typ == 2:  # regexp\n            name = RegExpName\n        elif typ == 3:  # number\n            name = NumberName\n        else:\n            raise RuntimeError()\n        res += ' ' + name % count + ' '\n        constants[name % count] = source[end:next_start]\n        count += 1\n    res += source[start:]\n    # remove this stupid white space\n    for e in WHITE:\n        res = res.replace(e, ' ')\n    res = res.replace(CR + LF, '\\n')\n    for e in LINE_TERMINATOR:\n        res = res.replace(e, '\\n')\n    return res.strip(), constants"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef recover_constants(py_source,\n                      replacements):  #now has n^2 complexity. improve to n\n    '''Converts identifiers representing Js constants to the PyJs constants\n    PyJsNumberConst_1_ which has the true value of 5 will be converted to PyJsNumber(5)'''\n    for identifier, value in replacements.iteritems():\n        if identifier.startswith('PyJsConstantRegExp'):\n            py_source = py_source.replace(identifier,\n                                          'JsRegExp(%s)' % repr(value))\n        elif identifier.startswith('PyJsConstantString'):\n            py_source = py_source.replace(\n                identifier, 'Js(u%s)' % unify_string_literals(value))\n        else:\n            py_source = py_source.replace(identifier, 'Js(%s)' % value)\n    return py_source", "response": "Converts identifiers representing Js constants to the PyJs constants\n    PyJsNumberConst_1_ which has the true value of 5 will be converted to PyJsNumber ( 5 )"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef unify_string_literals(js_string):\n    n = 0\n    res = ''\n    limit = len(js_string)\n    while n < limit:\n        char = js_string[n]\n        if char == '\\\\':\n            new, n = do_escape(js_string, n)\n            res += new\n        else:\n            res += char\n            n += 1\n    return res", "response": "this function parses the string just like javascript\n       for example literal \\ d in JavaScript would be interpreted\n       as '\\\\d' - backslash would be ignored"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef abstract_relational_comparison(self, other,\n                                   self_first=True):  # todo speed up!\n    ''' self<other if self_first else other<self.\n       Returns the result of the question: is self smaller than other?\n       in case self_first is false it returns the answer of:\n                                           is other smaller than self.\n       result is PyJs type: bool or undefined'''\n\n    px = to_primitive(self, 'Number')\n    py = to_primitive(other, 'Number')\n    if not self_first:  # reverse order\n        px, py = py, px\n    if not (Type(px) == 'String' and Type(py) == 'String'):\n        px, py = to_number(px), to_number(py)\n        if is_nan(px) or is_nan(py):\n            return None  # watch out here!\n        return px < py  # same cmp algorithm\n    else:\n        # I am pretty sure that python has the same\n        # string cmp algorithm but I have to confirm it\n        return px < py", "response": "abstract comparison of two objects"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the result of JS == compare. js == compare. js == compare. js == compare. js == compare. js == compare. js == compare. js == compare. js == compare. js == compare. js == compare. js == compare. js == compare. js", "response": "def abstract_equality_op(self, other):\n    ''' returns the result of JS == compare.\n       result is PyJs type: bool'''\n    tx, ty = Type(self), Type(other)\n    if tx == ty:\n        if tx == 'Undefined' or tx == 'Null':\n            return True\n        if tx == 'Number' or tx == 'String' or tx == 'Boolean':\n            return self == other\n        return self is other  # Object\n    elif (tx == 'Undefined' and ty == 'Null') or (ty == 'Undefined'\n                                                  and tx == 'Null'):\n        return True\n    elif tx == 'Number' and ty == 'String':\n        return abstract_equality_op(self, to_number(other))\n    elif tx == 'String' and ty == 'Number':\n        return abstract_equality_op(to_number(self), other)\n    elif tx == 'Boolean':\n        return abstract_equality_op(to_number(self), other)\n    elif ty == 'Boolean':\n        return abstract_equality_op(self, to_number(other))\n    elif (tx == 'String' or tx == 'Number') and is_object(other):\n        return abstract_equality_op(self, to_primitive(other))\n    elif (ty == 'String' or ty == 'Number') and is_object(self):\n        return abstract_equality_op(to_primitive(self), other)\n    else:\n        return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks if self is in other", "response": "def in_op(self, other):\n    '''checks if self is in other'''\n    if not is_object(other):\n        raise MakeError(\n            'TypeError',\n            \"You can\\'t use 'in' operator to search in non-objects\")\n    return other.has_property(to_string(self))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef maybe_download_and_extract():\n  dest_directory = '.'\n  filename = DATA_URL.split('/')[-1]\n  filepath = os.path.join(dest_directory, filename)\n  if not os.path.exists(filepath):\n    def _progress(count, block_size, total_size):\n      sys.stdout.write('\\r>> Downloading %s %.1f%%' % (filename,\n          float(count * block_size) / float(total_size) * 100.0))\n      sys.stdout.flush()\n    filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress)\n    print()\n    statinfo = os.stat(filepath)\n    print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')\n  extracted_dir_path = os.path.join(dest_directory, 'trees')\n  if not os.path.exists(extracted_dir_path):\n    zip_ref = zipfile.ZipFile(filepath, 'r')\n    zip_ref.extractall(dest_directory)\n    zip_ref.close()", "response": "Download and extract processed data and embeddings."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef make_network_graph(compact, expression_names, lookup_names):\n  nodes = set()\n#   edges = defaultdict(set) # parent -> (child, extra)\n  \n  var_name_dict = dict()\n  if expression_names:\n    for e in graphviz_items: # e: Expression\n      if e in expression_names:\n        var_name_dict[e.vindex] = expression_names[e]\n  \n  rnn_bldr_name = defaultdict(lambda: chr(len(rnn_bldr_name)+ord('A')))\n  def vidx2str(vidx): return '%s%s' % ('N', vidx)\n\n  for e in graphviz_items: # e: Expression\n    vidx = e.vindex\n    f_name = e.name\n    args = e.args\n    output_dim = e.dim\n    input_dim = None # basically just RNNStates use this since everything else has input_dim==output_dim\n    children = set()\n    node_type = '2_regular'\n    \n    if f_name == 'vecInput':\n      [_dim] = args\n      arg_strs = []\n    elif f_name == 'inputVector':\n      [_v] = args\n      arg_strs = []\n    elif f_name == 'matInput':\n      [_d1, _d2] = args\n      arg_strs = []\n    elif f_name == 'inputMatrix':\n      [_v, _d] = args\n      arg_strs = []\n    elif f_name == 'parameters':\n      [_dim] = args\n      arg_strs = []\n      if compact:\n        if vidx in var_name_dict:\n          f_name = var_name_dict[vidx]\n      node_type = '1_param'\n    elif f_name == 'lookup_parameters':\n      [_dim] = args\n      arg_strs = []\n      if compact:\n        if vidx in var_name_dict:\n          f_name = var_name_dict[vidx]\n      node_type = '1_param'\n    elif f_name == 'lookup':\n      [p, idx, update] = args\n      [_dim] = p.args\n      if vidx in var_name_dict:\n        name = var_name_dict[vidx]\n      else:\n        name = None\n      item_name = None\n      if lookup_names and p in expression_names:\n        param_name = expression_names[p]\n        if param_name in lookup_names:\n          item_name = '\\\\\"%s\\\\\"' % (lookup_names[param_name][idx],)\n      if compact:\n        if item_name is not None:\n          f_name = item_name\n        elif name is not None:\n          f_name = '%s[%s]' % (name, idx)\n        else:\n          f_name = 'lookup(%s)' % (idx)\n        arg_strs = []\n      else:\n        arg_strs = [var_name_dict.get(p.vindex, 'v%d' % (p.vindex))]\n        if item_name is not None:\n          arg_strs.append(item_name)\n        vocab_size = _dim[0]\n        arg_strs.extend(['%s' % (idx), '%s' % (vocab_size), 'update' if update else 'fixed'])\n      #children.add(vidx2str(p.vindex))\n      #node_type = '1_param'\n    elif f_name == 'RNNState':\n      [arg, input_dim, bldr_type, bldr_num, state_idx] = args # arg==input_e\n      rnn_name = rnn_bldr_name[bldr_num]\n      if bldr_type.endswith('Builder'):\n        bldr_type[:-len('Builder')]\n      f_name = '%s-%s-%s' % (bldr_type, rnn_name, state_idx)\n      if not compact:\n        i = arg.vindex\n        s = var_name_dict.get(i, 'v%d' % (i))\n        arg_strs = [s]\n      else:\n        arg_strs = []\n      children.add(vidx2str(arg.vindex))\n      node_type = '3_rnn_state'\n    else:\n      arg_strs = []\n      for arg in args:\n        if isinstance(arg, Expression):\n          if not compact:\n            i = arg.vindex\n            s = var_name_dict.get(i, 'v%d' % (i))\n            arg_strs.append(s)\n          children.add(vidx2str(arg.vindex))\n        elif isinstance(arg, float) and compact:\n          s = re.sub('0+$', '', '%.3f' % (arg))\n          if s == '0.':\n            s = str(arg)\n          arg_strs.append(s)\n        else:\n          arg_strs.append(str(arg))\n        \n#     f_name = { ,\n#              }.get(f_name, f_name)\n      \n    if compact:\n      f_name = { 'add': '+',\n                 'sub': '-',\n                 'mul': '*',\n                 'div': '/',\n                 'cadd': '+',\n                 'cmul': '*',\n                 'cdiv': '/',\n                 'scalarsub': '-',\n                 'concatenate': 'cat',\n                 'esum': 'sum',\n                 'emax': 'max',\n                 'emin': 'min',\n               }.get(f_name, f_name)\n      if arg_strs:\n        str_repr = '%s(%s)' % (f_name, ', '.join(arg_strs))\n      else:\n        str_repr = f_name\n    elif f_name == 'add':\n      [a,b] = arg_strs\n      str_repr = '%s + %s' % (a,b)\n    elif f_name == 'sub':\n      [a,b] = arg_strs\n      str_repr = '%s - %s' % (a,b)\n    elif f_name == 'mul':\n      [a,b] = arg_strs\n      str_repr = '%s * %s' % (a,b)\n    elif f_name == 'div':\n      [a,b] = arg_strs\n      str_repr = '%s / %s' % (a,b)\n    elif f_name == 'neg':\n      [a,] = arg_strs\n      str_repr = '-%s' % (a)\n    elif f_name == 'affine_transform':\n      str_repr = arg_strs[0]\n      for i in xrange(1, len(arg_strs), 2):\n        str_repr += ' + %s*%s' % tuple(arg_strs[i:i+2])\n    else:\n      if arg_strs is not None:\n        str_repr = '%s(%s)' % (f_name, ', '.join(arg_strs))\n      else:\n        str_repr = f_name\n        \n    name = vidx2str(vidx)\n    var_name = '%s' % (var_name_dict.get(vidx, 'v%d' % (vidx))) if not compact else ''\n#     if show_dims:\n#       str_repr = '%s\\\\n%s' % (shape_str(e.dim), str_repr)\n    label = str_repr\n    if not compact:\n      label = '%s = %s' % (var_name, label)\n    features = ''\n#     if output_dim.invalid():\n#       features += \" [color=red,style=filled,fillcolor=red]\"\n#     node_def_lines.append('  %s [label=\"%s%s\"] %s;' % (vidx2str(vidx), label_prefix, str_repr, ''))\n    expr_name = expression_names[e] if compact and expression_names and (e in expression_names) and (expression_names[e] != f_name) else None\n    nodes.add(GVNode(name, input_dim, label, output_dim, frozenset(children), features, node_type, expr_name))\n\n  return nodes", "response": "Make a network graph from a set of expression names and a set of lookup names."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef add_inputs(self, es):\n        for e in es:\n            ensure_freshness(e)\n        for (fb,bb) in self.builder_layers[:-1]:\n            fs = fb.initial_state().transduce(es)\n            bs = bb.initial_state().transduce(reversed(es))\n            es = [concatenate([f,b]) for f,b in zip(fs, reversed(bs))]\n        (fb,bb) = self.builder_layers[-1]\n        fs = fb.initial_state().add_inputs(es)\n        bs = bb.initial_state().add_inputs(reversed(es))\n        return [(f,b) for f,b in zip(fs, reversed(bs))]", "response": "returns the list of state pairs obtained by adding \n        inputs to both forward and backward states."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the list of output Expressions obtained by adding the given inputs to the current state, one by one, to both the forward and backward RNNs, and concatenating. @param es: a list of Expression see also add_inputs(xs) .transduce(xs) is different from .add_inputs(xs) in the following way: .add_inputs(xs) returns a list of RNNState pairs. RNNState objects can be queried in various ways. In particular, they allow access to the previous state, as well as to the state-vectors (h() and s() ) .transduce(xs) returns a list of Expression. These are just the output expressions. For many cases, this suffices. transduce is much more memory efficient than add_inputs.", "response": "def transduce(self, es):\n        \"\"\"\n        returns the list of output Expressions obtained by adding the given inputs\n        to the current state, one by one, to both the forward and backward RNNs, \n        and concatenating.\n        \n        @param es: a list of Expression\n\n        see also add_inputs(xs)\n\n        .transduce(xs) is different from .add_inputs(xs) in the following way:\n\n            .add_inputs(xs) returns a list of RNNState pairs. RNNState objects can be\n             queried in various ways. In particular, they allow access to the previous\n             state, as well as to the state-vectors (h() and s() )\n\n            .transduce(xs) returns a list of Expression. These are just the output\n             expressions. For many cases, this suffices. \n             transduce is much more memory efficient than add_inputs. \n        \"\"\"\n        for e in es:\n            ensure_freshness(e)\n        for (fb,bb) in self.builder_layers:\n            fs = fb.initial_state().transduce(es)\n            bs = bb.initial_state().transduce(reversed(es))\n            es = [concatenate([f,b]) for f,b in zip(fs, reversed(bs))]\n        return es"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef add_inputs(self, xs):\n        states = []\n        cur = self\n        for x in xs:\n            cur = cur.add_input(x)\n            states.append(cur)\n        return states", "response": "Returns the list of states obtained by adding the given inputs\n        to the current state one by one."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef load_mnist(dataset, path):\n    if dataset is \"training\":\n        fname_img = os.path.join(path, \"train-images-idx3-ubyte\")\n        fname_lbl = os.path.join(path, \"train-labels-idx1-ubyte\")\n    elif dataset is \"testing\":\n        fname_img = os.path.join(path, \"t10k-images-idx3-ubyte\")\n        fname_lbl = os.path.join(path, \"t10k-labels-idx1-ubyte\")\n    else:\n        raise ValueError(\"dataset must be 'testing' or 'training'\")\n\n    # Load everything in numpy arrays\n    with open(fname_lbl, \"rb\") as flbl:\n        magic, num = struct.unpack(\">II\", flbl.read(8))\n        labels = np.fromfile(flbl, dtype=np.int8)\n\n    with open(fname_img, \"rb\") as fimg:\n        magic, num, rows, cols = struct.unpack(\">IIII\", fimg.read(16))\n        images = np.multiply(\n            np.fromfile(fimg, dtype=np.uint8).reshape(len(labels), rows*cols),\n            1.0 / 255.0)\n\n    get_instance = lambda idx: (labels[idx], images[idx].reshape(1, 28, 28))\n\n    # Create an iterator which returns each image in turn\n    # for i in range(len(labels)):\n    #     yield get_instance(i)\n\n    size_reset = lambda x: x.reshape(1, 28, 28)\n    return list(map(size_reset, images))", "response": "Load the mnist dataset from the given path."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nmaking a grid of images from a 4D mini - batch tensor.", "response": "def make_grid(tensor, nrow=8, padding=2, pad_value=0):\n    \"\"\"Make a grid of images, via numpy.\n\n    Args:\n        tensor (Tensor or list): 4D mini-batch Tensor of shape (B x C x H x W)\n            or a list of images all of the same size.\n        nrow (int, optional): Number of images displayed in each row of the grid.\n            The Final grid size is (B / nrow, nrow). Default is 8.\n        padding (int, optional): amount of padding. Default is 2.\n        pad_value (float, optional): Value for the padded pixels.\n\n    \"\"\"\n    if not (isinstance(tensor, np.ndarray) or\n            (isinstance(tensor, list) and all(isinstance(t, np.ndarray) for t in tensor))):\n        raise TypeError('tensor or list of tensors expected, got {}'.format(type(tensor)))\n\n    # if list of tensors, convert to a 4D mini-batch Tensor\n    if isinstance(tensor, list):\n        tensor = np.stack(tensor, 0)\n\n    if tensor.ndim == 2:  # single image H x W\n        tensor = tensor.reshape((1, tensor.shape[0], tensor.shape[1]))\n\n    if tensor.ndim == 3:\n        if tensor.shape[0] == 1:  # if single-channel, single image, convert to 3-channel\n            tensor = np.concatenate((tensor, tensor, tensor), 0)\n        tensor = tensor.reshape((1, tensor.shape[0], tensor.shape[1], tensor.shape[2]))\n\n    if tensor.ndim == 4 and tensor.shape[1] == 1:  # single-channel images\n        tensor = np.concatenate((tensor, tensor, tensor), 1)\n\n    if tensor.shape[0] == 1:\n        return np.squeeze(tensor)\n\n    # make the mini-batch of images into a grid\n    nmaps = tensor.shape[0]\n    xmaps = min(nrow, nmaps)\n    ymaps = int(math.ceil(float(nmaps) / xmaps))\n    height, width = int(tensor.shape[2] + padding), int(tensor.shape[3] + padding)\n    grid = np.ones((3, height * ymaps + padding, width * xmaps + padding)) * pad_value\n    k = 0\n    for y in range(ymaps):\n        for x in range(xmaps):\n            if k >= nmaps:\n                break\n            grid[:, y * height + padding:(y+1) * height,\\\n                 x * width + padding:(x+1) * width] = tensor[k]\n            k = k + 1\n    return grid"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef save_image(tensor, filename, nrow=8, padding=2, pad_value=0):\n    from PIL import Image\n    grid = make_grid(tensor, nrow=nrow, padding=padding, pad_value=pad_value)\n    im = Image.fromarray(pre_pillow_float_img_process(grid))\n    im.save(filename)", "response": "Save a given Tensor into an image file."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nconverts a string of arguments into a list of strings.", "response": "def pythonize_arguments(arg_str):\n    \"\"\"\n    Remove types from function arguments in cython\n    \"\"\"\n    out_args = []\n    # If there aren't any arguments return the empty string\n    if arg_str is None:\n        return out_str\n    args = arg_str.split(',')\n    for arg in args:\n        components = arg.split('=')\n        name_and_type=components[0].split(' ')\n        # There is probably type info\n        if name_and_type[-1]=='' and len(name_and_type)>1:\n            name=name_and_type[-2]\n        else:\n            name=name_and_type[-1]\n        # if there are default parameters\n        if len(components)>1:\n            name+='='+components[1]\n\n        out_args.append(name)\n    return ','.join(out_args)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncommunicating with the child process without closing stdin.", "response": "def correspond(text):\n    \"\"\"Communicate with the child process without closing stdin.\"\"\"\n    if text:\n        subproc.stdin.write(text)\n    subproc.stdin.flush()\n    return get_lines()"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _normalize(c):\n    if isinstance(c, int):\n        return bytes([c])\n    elif isinstance(c, str):\n        return bytes([ord(c)])\n    else:\n        return c", "response": "Convert a byte - like value into a canonical byte - like value."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _set_perms(self, perms):\n        assert isinstance(perms, str) and len(perms) <= 3 and perms.strip() in ['', 'r', 'w', 'x', 'rw', 'r x', 'rx', 'rwx', 'wx', ]\n        self._perms = perms", "response": "Sets the permissions of the current object."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks if the user has enough permissions for access", "response": "def access_ok(self, access):\n        \"\"\" Check if there is enough permissions for access \"\"\"\n        for c in access:\n            if c not in self.perms:\n                return False\n        return True"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn True if index is in range False otherwise.", "response": "def _in_range(self, index):\n        \"\"\" Returns True if index is in range \"\"\"\n        if isinstance(index, slice):\n            in_range = index.start < index.stop and \\\n                index.start >= self.start and \\\n                index.stop <= self.end\n        else:\n            in_range = index >= self.start and \\\n                index <= self.end\n        return in_range"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ntranslates the index to the internal offsets.", "response": "def _get_offset(self, index):\n        \"\"\"\n        Translates the index to the internal offsets.\n\n        self.start   -> 0\n        self.start+1 -> 1\n        ...\n        self.end     -> len(self)\n        \"\"\"\n        if not self._in_range(index):\n            raise IndexError('Map index out of range')\n        if isinstance(index, slice):\n            index = slice(index.start - self.start, index.stop - self.start)\n        else:\n            index -= self.start\n        return index"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the ceil value of the given address.", "response": "def _ceil(self, address):\n        \"\"\"\n        Returns the smallest page boundary value not less than the address.\n        :rtype: int\n        :param address: the address to calculate its ceil.\n        :return: the ceil of C{address}.\n        \"\"\"\n        return (((address - 1) + self.page_size) & ~self.page_mask) & self.memory_mask"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a new file mapping in the memory address space.", "response": "def mmapFile(self, addr, size, perms, filename, offset=0):\n        \"\"\"\n        Creates a new file mapping in the memory address space.\n\n        :param addr: the starting address (took as hint). If C{addr} is C{0} the first big enough\n                     chunk of memory will be selected as starting address.\n        :param size: the contents of a file mapping are initialized using C{size} bytes starting\n                     at offset C{offset} in the file C{filename}.\n        :param perms: the access permissions to this memory.\n        :param filename: the pathname to the file to map.\n        :param offset: the contents of a file mapping are initialized using C{size} bytes starting\n                      at offset C{offset} in the file C{filename}.\n        :return: the starting address where the file was mapped.\n        :rtype: int\n        :raises error:\n                   - 'Address shall be concrete' if C{addr} is not an integer number.\n                   - 'Address too big' if C{addr} goes beyond the limit of the memory.\n                   - 'Map already used' if the piece of memory starting in C{addr} and with length C{size} isn't free.\n        \"\"\"\n        # If addr is NULL, the system determines where to allocate the region.\n        assert addr is None or isinstance(addr, int), 'Address shall be concrete'\n        assert size > 0\n\n        self.cpu._publish('will_map_memory', addr, size, perms, filename, offset)\n\n        # address is rounded down to the nearest multiple of the allocation granularity\n        if addr is not None:\n            assert addr < self.memory_size, 'Address too big'\n            addr = self._floor(addr)\n\n        # size value is rounded up to the next page boundary\n        size = self._ceil(size)\n\n        # If zero search for a spot\n        addr = self._search(size, addr)\n\n        # It should not be allocated\n        for i in range(self._page(addr), self._page(addr + size)):\n            assert i not in self._page2map, 'Map already used'\n\n        # Create the map\n        m = FileMap(addr, size, perms, filename, offset)\n\n        # Okay, ready to alloc\n        self._add(m)\n\n        logger.debug('New file-memory map @%x size:%x', addr, size)\n        self.cpu._publish('did_map_memory', addr, size, perms, filename, offset, addr)\n        return addr"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef mmap(self, addr, size, perms, data_init=None, name=None):\n        # If addr is NULL, the system determines where to allocate the region.\n        assert addr is None or isinstance(addr, int), 'Address shall be concrete'\n\n        self.cpu._publish('will_map_memory', addr, size, perms, None, None)\n\n        # address is rounded down to the nearest multiple of the allocation granularity\n        if addr is not None:\n            assert addr < self.memory_size, 'Address too big'\n            addr = self._floor(addr)\n\n        # size value is rounded up to the next page boundary\n        size = self._ceil(size)\n\n        # If zero search for a spot\n        addr = self._search(size, addr)\n\n        # It should not be allocated\n        for i in range(self._page(addr), self._page(addr + size)):\n            assert i not in self._page2map, 'Map already used'\n\n        # Create the anonymous map\n        m = AnonMap(start=addr, size=size, perms=perms, data_init=data_init, name=name)\n\n        # Okay, ready to alloc\n        self._add(m)\n\n        logger.debug('New memory map @%x size:%x', addr, size)\n\n        self.cpu._publish('did_map_memory', addr, size, perms, None, None, addr)\n        return addr", "response": "Creates a new mapping in the memory address space."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the object containing the address.", "response": "def map_containing(self, address):\n        \"\"\"\n        Returns the L{MMap} object containing the address.\n\n        :param address: the address to obtain its mapping.\n        :rtype: L{MMap}\n\n        @todo: symbolic address\n        \"\"\"\n        page_offset = self._page(address)\n        if page_offset not in self._page2map:\n            raise MemoryException(\"Page not mapped\", address)\n        return self._page2map[page_offset]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a sorted list of all the mappings for this memory.", "response": "def mappings(self):\n        \"\"\"\n        Returns a sorted list of all the mappings for this memory.\n\n        :return: a list of mappings.\n        :rtype: list\n        \"\"\"\n        result = []\n        for m in self.maps:\n            if isinstance(m, AnonMap):\n                result.append((m.start, m.end, m.perms, 0, ''))\n            elif isinstance(m, FileMap):\n                result.append((m.start, m.end, m.perms, m._offset, m._filename))\n            else:\n                result.append((m.start, m.end, m.perms, 0, m.name))\n\n        return sorted(result)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _maps_in_range(self, start, end):\n\n        # Search for the first matching map\n        addr = start\n        while addr < end:\n            if addr not in self:\n                addr += self.page_size\n            else:\n                m = self._page2map[self._page(addr)]\n                yield m\n                addr = m.end", "response": "Generate the list of maps that overlaps with the range [ start end )"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef munmap(self, start, size):\n        start = self._floor(start)\n        end = self._ceil(start + size)\n\n        self.cpu._publish('will_unmap_memory', start, size)\n\n        for m in self._maps_in_range(start, end):\n            self._del(m)\n            head, tail = m.split(start)\n            middle, tail = tail.split(end)\n            assert middle is not None\n            if head:\n                self._add(head)\n            if tail:\n                self._add(tail)\n\n        self.cpu._publish('did_unmap_memory', start, size)\n        logger.debug(f'Unmap memory @{start:x} size:{size:x}')", "response": "Removes mappings from the specified address range and causes further memory related references to addresses within the range to generate invalid memory."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef pop_record_writes(self):\n\n        lst = self._recording_stack.pop()\n        # Append the current list to a previously-started trace.\n        if self._recording_stack:\n            self._recording_stack[-1].extend(lst)\n        return lst", "response": "Stop recording trace and return a list[tuple ] of all the writes\n            that occurred where value is of type list[str ]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndeleting the mappings for the specified address range and causes further memory references to addresses within the range to generate invalid memory references.", "response": "def munmap(self, start, size):\n        \"\"\"\n        Deletes the mappings for the specified address range and causes further\n        references to addresses within the range to generate invalid memory\n        references.\n\n        :param start: the starting address to delete.\n        :param size: the length of the unmapping.\n        \"\"\"\n        for addr in range(start, start + size):\n            if len(self._symbols) == 0:\n                break\n            if addr in self._symbols:\n                del self._symbols[addr]\n        super().munmap(start, size)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading a stream of bytes from a potentially symbolic memory.", "response": "def read(self, address, size, force=False):\n        \"\"\"\n        Read a stream of potentially symbolic bytes from a potentially symbolic\n        address\n\n        :param address: Where to read from\n        :param size: How many bytes\n        :param force: Whether to ignore permissions\n        :rtype: list\n        \"\"\"\n        size = self._get_size(size)\n        assert not issymbolic(size)\n\n        if issymbolic(address):\n            assert solver.check(self.constraints)\n            logger.debug(f'Reading {size} bytes from symbolic address {address}')\n            try:\n                solutions = self._try_get_solutions(address, size, 'r', force=force)\n                assert len(solutions) > 0\n            except TooManySolutions as e:\n                m, M = solver.minmax(self.constraints, address)\n                logger.debug(f'Got TooManySolutions on a symbolic read. Range [{m:x}, {M:x}]. Not crashing!')\n\n                # The force param shouldn't affect this, as this is checking for unmapped reads, not bad perms\n                crashing_condition = True\n                for start, end, perms, offset, name in self.mappings():\n                    if start <= M + size and end >= m:\n                        if 'r' in perms:\n                            crashing_condition = Operators.AND(Operators.OR((address + size).ult(start), address.uge(end)), crashing_condition)\n\n                if solver.can_be_true(self.constraints, crashing_condition):\n                    raise InvalidSymbolicMemoryAccess(address, 'r', size, crashing_condition)\n\n                # INCOMPLETE Result! We could also fork once for every map\n                logger.info('INCOMPLETE Result! Using the sampled solutions we have as result')\n                condition = False\n                for base in e.solutions:\n                    condition = Operators.OR(address == base, condition)\n                from .state import ForkState\n                raise ForkState(\"Forking state on incomplete result\", condition)\n\n            # So here we have all potential solutions to address\n\n            condition = False\n            for base in solutions:\n                condition = Operators.OR(address == base, condition)\n\n            result = []\n            # consider size ==1 to read following code\n            for offset in range(size):\n                # Given ALL solutions for the symbolic address\n                for base in solutions:\n                    addr_value = base + offset\n                    byte = Operators.ORD(self.map_containing(addr_value)[addr_value])\n                    if addr_value in self._symbols:\n                        for condition, value in self._symbols[addr_value]:\n                            byte = Operators.ITEBV(8, condition, Operators.ORD(value), byte)\n                    if len(result) > offset:\n                        result[offset] = Operators.ITEBV(8, address == base, byte, result[offset])\n                    else:\n                        result.append(byte)\n                    assert len(result) == offset + 1\n            return list(map(Operators.CHR, result))\n        else:\n            result = list(map(Operators.ORD, super().read(address, size, force)))\n            for offset in range(size):\n                if address + offset in self._symbols:\n                    for condition, value in self._symbols[address + offset]:\n                        if condition is True:\n                            result[offset] = Operators.ORD(value)\n                        else:\n                            result[offset] = Operators.ITEBV(8, condition, Operators.ORD(value), result[offset])\n            return list(map(Operators.CHR, result))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwriting a value at the specified address.", "response": "def write(self, address, value, force=False):\n        \"\"\"\n        Write a value at address.\n        :param address: The address at which to write\n        :type address: int or long or Expression\n        :param value: Bytes to write\n        :type value: str or list\n        :param force: Whether to ignore permissions\n        \"\"\"\n        size = len(value)\n        if issymbolic(address):\n\n            solutions = self._try_get_solutions(address, size, 'w', force=force)\n\n            for offset in range(size):\n                for base in solutions:\n                    condition = base == address\n                    self._symbols.setdefault(base + offset, []).append((condition, value[offset]))\n        else:\n\n            for offset in range(size):\n                if issymbolic(value[offset]):\n                    if not self.access_ok(address + offset, 'w', force):\n                        raise InvalidMemoryAccess(address + offset, 'w')\n                    self._symbols[address + offset] = [(True, value[offset])]\n                else:\n                    # overwrite all previous items\n                    if address + offset in self._symbols:\n                        del self._symbols[address + offset]\n                    super().write(address + offset, [value[offset]], force)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _try_get_solutions(self, address, size, access, max_solutions=0x1000, force=False):\n        assert issymbolic(address)\n\n        solutions = solver.get_all_values(self.constraints, address, maxcnt=max_solutions)\n\n        crashing_condition = False\n        for base in solutions:\n            if not self.access_ok(slice(base, base + size), access, force):\n                crashing_condition = Operators.OR(address == base, crashing_condition)\n\n        if solver.can_be_true(self.constraints, crashing_condition):\n            raise InvalidSymbolicMemoryAccess(address, access, size, crashing_condition)\n\n        return solutions", "response": "Try to find a set of possible memory entries for a given symbolic address."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef mmapFile(self, addr, size, perms, filename, offset=0):\n        # If addr is NULL, the system determines where to allocate the region.\n        assert addr is None or isinstance(addr, int), 'Address shall be concrete'\n        assert addr < self.memory_size, 'Address too big'\n        assert size > 0\n\n        self.cpu._publish('will_map_memory', addr, size, perms, filename, offset)\n\n        map = AnonMap(addr, size, perms)\n        self._add(map)\n\n        # address is rounded down to the nearest multiple of the allocation granularity\n        if addr is not None:\n            addr = self._floor(addr)\n\n        # size value is rounded up to the next page boundary\n        size = self._ceil(size)\n\n        with open(filename, 'rb') as f:\n            fdata = f.read()  # fdata is a bytes now\n\n        # this worked\n        fdata = fdata[offset:]\n        fdata = fdata.ljust(size, b'\\0')\n\n        for i in range(size):\n            Memory.write(self, addr + i, chr(fdata[i]), force=True)\n\n        logger.debug('New file-memory map @%x size:%x', addr, size)\n        self.cpu._publish('did_map_memory', addr, size, perms, filename, offset, addr)\n\n        return addr", "response": "Creates a new file mapping in the memory address space."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nimporting concrete memory from from_addr to to_addr.", "response": "def _import_concrete_memory(self, from_addr, to_addr):\n        \"\"\"\n        for each address in this range need to read from concrete and write to symbolic\n        it's possible that there will be invalid/unmapped addresses in this range. need to skip to next map if so\n        also need to mark all of these addresses as now in the symbolic store\n\n        :param int from_addr:\n        :param int to_addr:\n        :return:\n        \"\"\"\n        logger.debug(\"Importing concrete memory: {:x} - {:x} ({} bytes)\".format(from_addr, to_addr, to_addr - from_addr))\n\n        for m in self.maps:\n            span = interval_intersection(m.start, m.end, from_addr, to_addr)\n            if span is None:\n                continue\n\n            start, stop = span\n\n            for addr in range(start, stop):\n                if addr in self.backed_by_symbolic_store:\n                    continue\n\n                self.backing_array[addr] = Memory.read(self, addr, 1)[0]\n                self.backed_by_symbolic_store.add(addr)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nscans for concrete bytes in all mapped memory. Successively yield addresses of all matches.", "response": "def scan_mem(self, data_to_find):\n        \"\"\"\n        Scan for concrete bytes in all mapped memory. Successively yield addresses of all matches.\n\n        :param bytes data_to_find: String to locate\n        :return:\n        \"\"\"\n\n        # TODO: for the moment we just treat symbolic bytes as bytes that don't match.\n        # for our simple test cases right now, the bytes we're interested in scanning\n        # for will all just be there concretely\n        # TODO: Can probably do something smarter here like Boyer-Moore, but unnecessary\n        # if we're looking for short strings.\n\n        # Querying mem with an index returns [bytes]\n        if isinstance(data_to_find, bytes):\n            data_to_find = [bytes([c]) for c in data_to_find]\n\n        for mapping in sorted(self.maps):\n            for ptr in mapping:\n                if ptr + len(data_to_find) >= mapping.end:\n                    break\n\n                candidate = mapping[ptr:ptr + len(data_to_find)]\n\n                # TODO: treat symbolic bytes as bytes that don't match. for our simple tests right now, the\n                # bytes will be there concretely\n                if issymbolic(candidate[0]):\n                    break\n\n                if candidate == data_to_find:\n                    yield ptr"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _reg_name(self, reg_id):\n        if reg_id >= X86_REG_ENDING:\n            logger.warning(\"Trying to get register name for a non-register\")\n            return None\n        cs_reg_name = self.cpu.instruction.reg_name(reg_id)\n        if cs_reg_name is None or cs_reg_name.lower() == '(invalid)':\n            return None\n        return self.cpu._regfile._alias(cs_reg_name.upper())", "response": "Translate a register ID into a register name based on the alias in the register file."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_argument_values(self, model, prefix_args):\n        spec = inspect.getfullargspec(model)\n\n        if spec.varargs:\n            logger.warning(\"ABI: A vararg model must be a unary function.\")\n\n        nargs = len(spec.args) - len(prefix_args)\n\n        # If the model is a method, we need to account for `self`\n        if inspect.ismethod(model):\n            nargs -= 1\n\n        def resolve_argument(arg):\n            if isinstance(arg, str):\n                return self._cpu.read_register(arg)\n            else:\n                return self._cpu.read_int(arg)\n\n        # Create a stream of resolved arguments from argument descriptors\n        descriptors = self.get_arguments()\n        argument_iter = map(resolve_argument, descriptors)\n\n        from ..models import isvariadic  # prevent circular imports\n\n        if isvariadic(model):\n            arguments = prefix_args + (argument_iter,)\n        else:\n            arguments = prefix_args + tuple(islice(argument_iter, nargs))\n\n        return arguments", "response": "Extract the arguments for the given model and return as a tuple that is ready to be passed to the model."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninvokes a native function of the class.", "response": "def invoke(self, model, prefix_args=None):\n        \"\"\"\n        Invoke a callable `model` as if it was a native function. If\n        :func:`~manticore.models.isvariadic` returns true for `model`, `model` receives a single\n        argument that is a generator for function arguments. Pass a tuple of\n        arguments for `prefix_args` you'd like to precede the actual\n        arguments.\n\n        :param callable model: Python model of the function\n        :param tuple prefix_args: Parameters to pass to model before actual ones\n        :return: The result of calling `model`\n        \"\"\"\n        prefix_args = prefix_args or ()\n\n        arguments = self.get_argument_values(model, prefix_args)\n\n        try:\n            result = model(*arguments)\n        except ConcretizeArgument as e:\n            assert e.argnum >= len(prefix_args), \"Can't concretize a constant arg\"\n            idx = e.argnum - len(prefix_args)\n\n            # Arguments were lazily computed in case of variadic, so recompute here\n            descriptors = self.get_arguments()\n            src = next(islice(descriptors, idx, idx + 1))\n\n            msg = 'Concretizing due to model invocation'\n            if isinstance(src, str):\n                raise ConcretizeRegister(self._cpu, src, msg)\n            else:\n                raise ConcretizeMemory(self._cpu.memory, src, self._cpu.address_bit_size, msg)\n        else:\n            if result is not None:\n                self.write_result(result)\n\n            self.ret()\n\n        return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef write_register(self, register, value):\n        self._publish('will_write_register', register, value)\n        value = self._regfile.write(register, value)\n        self._publish('did_write_register', register, value)\n        return value", "response": "Dynamic interface for writing cpu registers"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef emulate_until(self, target: int):\n        self._concrete = True\n        self._break_unicorn_at = target\n        if self.emu:\n            self.emu._stop_at = target", "response": "Sets up a concrete unicorn emulator and use it to execute instructions until target is reached."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef write_int(self, where, expression, size=None, force=False):\n        if size is None:\n            size = self.address_bit_size\n        assert size in SANE_SIZES\n        self._publish('will_write_memory', where, expression, size)\n\n        data = [Operators.CHR(Operators.EXTRACT(expression, offset, 8)) for offset in range(0, size, 8)]\n        self._memory.write(where, data, force)\n\n        self._publish('did_write_memory', where, expression, size)", "response": "Writes an integer to the memory."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nselect bytes from memory. Attempts to do so faster than via read_bytes. :param where: address to read from :param size: number of bytes to read :return: the bytes in memory", "response": "def _raw_read(self, where: int, size=1) -> bytes:\n        \"\"\"\n        Selects bytes from memory. Attempts to do so faster than via read_bytes.\n\n        :param where: address to read from\n        :param size: number of bytes to read\n        :return: the bytes in memory\n        \"\"\"\n        map = self.memory.map_containing(where)\n        start = map._get_offset(where)\n        mapType = type(map)\n        if mapType is FileMap:\n            end = map._get_offset(where + size)\n\n            if end > map._mapped_size:\n                logger.warning(f\"Missing {end - map._mapped_size} bytes at the end of {map._filename}\")\n\n            raw_data = map._data[map._get_offset(where): min(end, map._mapped_size)]\n            if len(raw_data) < end:\n                raw_data += b'\\x00' * (end - len(raw_data))\n\n            data = b''\n            for offset in sorted(map._overlay.keys()):\n                data += raw_data[len(data):offset]\n                data += map._overlay[offset]\n            data += raw_data[len(data):]\n\n        elif mapType is AnonMap:\n            data = bytes(map._data[start:start + size])\n        else:\n            data = b''.join(self.memory[where:where + size])\n        assert len(data) == size, 'Raw read resulted in wrong data read which should never happen'\n        return data"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef read_int(self, where, size=None, force=False):\n        if size is None:\n            size = self.address_bit_size\n        assert size in SANE_SIZES\n        self._publish('will_read_memory', where, size)\n\n        data = self._memory.read(where, size // 8, force)\n        assert (8 * len(data)) == size\n        value = Operators.CONCAT(size, *map(Operators.ORD, reversed(data)))\n\n        self._publish('did_read_memory', where, value, size)\n        return value", "response": "Reads an integer from memory returning it as a BitVec."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef write_bytes(self, where, data, force=False):\n\n        mp = self.memory.map_containing(where)\n        # TODO (ehennenfent) - fast write can have some yet-unstudied unintended side effects.\n        # At the very least, using it in non-concrete mode will break the symbolic strcmp/strlen models. The 1024 byte\n        # minimum is intended to minimize the potential effects of this by ensuring that if there _are_ any other\n        # issues, they'll only crop up when we're doing very large writes, which are fairly uncommon.\n        can_write_raw = type(mp) is AnonMap and \\\n            isinstance(data, (str, bytes)) and \\\n            (mp.end - mp.start + 1) >= len(data) >= 1024 and \\\n            not issymbolic(data) and \\\n            self._concrete\n\n        if can_write_raw:\n            logger.debug(\"Using fast write\")\n            offset = mp._get_offset(where)\n            if isinstance(data, str):\n                data = bytes(data.encode('utf-8'))\n            mp._data[offset:offset + len(data)] = data\n            self._publish('did_write_memory', where, data, 8 * len(data))\n        else:\n            for i in range(len(data)):\n                self.write_int(where + i, Operators.ORD(data[i]), 8, force)", "response": "Write a byte array to memory."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef read_bytes(self, where, size, force=False):\n        result = []\n        for i in range(size):\n            result.append(Operators.CHR(self.read_int(where + i, 8, force)))\n        return result", "response": "Reads a number of bytes from memory."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nwrites a string to memory.", "response": "def write_string(self, where, string, max_length=None, force=False):\n        \"\"\"\n        Writes a string to memory, appending a NULL-terminator at the end.\n        :param int where: Address to write the string to\n        :param str string: The string to write to memory\n        :param int max_length:\n            The size in bytes to cap the string at, or None [default] for no\n            limit. This includes the NULL terminator.\n        :param force: whether to ignore memory permissions\n        \"\"\"\n\n        if max_length is not None:\n            string = string[:max_length - 1]\n\n        self.write_bytes(where, string + '\\x00', force)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreads a NUL - terminated concrete buffer from memory. Stops reading at first symbolic byte.", "response": "def read_string(self, where, max_length=None, force=False):\n        \"\"\"\n        Read a NUL-terminated concrete buffer from memory. Stops reading at first symbolic byte.\n\n        :param int where: Address to read string from\n        :param int max_length:\n            The size in bytes to cap the string at, or None [default] for no\n            limit.\n        :param force: whether to ignore memory permissions\n        :return: string read\n        :rtype: str\n        \"\"\"\n        s = io.BytesIO()\n        while True:\n            c = self.read_int(where, 8, force)\n\n            if issymbolic(c) or c == 0:\n                break\n\n            if max_length is not None:\n                if max_length == 0:\n                    break\n                max_length = max_length - 1\n            s.write(Operators.CHR(c))\n            where += 1\n        return s.getvalue().decode()"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef push_bytes(self, data, force=False):\n        self.STACK -= len(data)\n        self.write_bytes(self.STACK, data, force)\n        return self.STACK", "response": "Writes data to the stack and decrement the stack pointer accordingly."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread nbytes bytes from the stack and return them.", "response": "def pop_bytes(self, nbytes, force=False):\n        \"\"\"\n        Read `nbytes` from the stack, increment the stack pointer, and return\n        data.\n\n        :param int nbytes: How many bytes to read\n        :param force: whether to ignore memory permissions\n        :return: Data read from the stack\n        \"\"\"\n        data = self.read_bytes(self.STACK, nbytes, force=force)\n        self.STACK += nbytes\n        return data"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef push_int(self, value, force=False):\n        self.STACK -= self.address_bit_size // 8\n        self.write_int(self.STACK, value, force=force)\n        return self.STACK", "response": "Decrement the stack pointer and write the given value to the stack."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading an integer from the stack and increment the stack pointer.", "response": "def pop_int(self, force=False):\n        \"\"\"\n        Read a value from the stack and increment the stack pointer.\n\n        :param force: whether to ignore memory permissions\n        :return: Value read\n        \"\"\"\n        value = self.read_int(self.STACK, force=force)\n        self.STACK += self.address_bit_size // 8\n        return value"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndecoding and execute one instruction pointed by register PC", "response": "def execute(self):\n        \"\"\"\n        Decode, and execute one instruction pointed by register PC\n        \"\"\"\n        if issymbolic(self.PC):\n            raise ConcretizeRegister(self, 'PC', policy='ALL')\n\n        if not self.memory.access_ok(self.PC, 'x'):\n            raise InvalidMemoryAccess(self.PC, 'x')\n\n        self._publish('will_decode_instruction', self.PC)\n\n        insn = self.decode_instruction(self.PC)\n        self._last_pc = self.PC\n\n        self._publish('will_execute_instruction', self.PC, insn)\n\n        # FIXME (theo) why just return here?\n        if insn.address != self.PC:\n            return\n\n        name = self.canonicalize_instruction_name(insn)\n\n        if logger.level == logging.DEBUG:\n            logger.debug(self.render_instruction(insn))\n            for l in self.render_registers():\n                register_logger.debug(l)\n\n        try:\n            if self._concrete and 'SYSCALL' in name:\n                self.emu.sync_unicorn_to_manticore()\n            if self._concrete and 'SYSCALL' not in name:\n                self.emulate(insn)\n                if self.PC == self._break_unicorn_at:\n                    logger.debug(\"Switching from Unicorn to Manticore\")\n                    self._break_unicorn_at = None\n                    self._concrete = False\n            else:\n                implementation = getattr(self, name, None)\n\n                if implementation is not None:\n                    implementation(*insn.operands)\n\n                else:\n                    text_bytes = ' '.join('%02x' % x for x in insn.bytes)\n                    logger.warning(\"Unimplemented instruction: 0x%016x:\\t%s\\t%s\\t%s\",\n                                   insn.address, text_bytes, insn.mnemonic, insn.op_str)\n                    self.backup_emulate(insn)\n        except (Interruption, Syscall) as e:\n            e.on_handled = lambda: self._publish_instruction_as_executed(insn)\n            raise e\n        else:\n            self._publish_instruction_as_executed(insn)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _publish_instruction_as_executed(self, insn):\n        self._icount += 1\n        self._publish('did_execute_instruction', self._last_pc, self.PC, insn)", "response": "Notify listeners that an instruction has been executed."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef emulate(self, insn):\n\n        if self._concrete:\n            self.concrete_emulate(insn)\n        else:\n            self.backup_emulate(insn)", "response": "Emulate the right instruction."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef concrete_emulate(self, insn):\n\n        if not self.emu:\n            self.emu = ConcreteUnicornEmulator(self)\n            self.emu._stop_at = self._break_unicorn_at\n        try:\n            self.emu.emulate(insn)\n        except unicorn.UcError as e:\n            if e.errno == unicorn.UC_ERR_INSN_INVALID:\n                text_bytes = ' '.join('%02x' % x for x in insn.bytes)\n                logger.error(\"Unimplemented instruction: 0x%016x:\\t%s\\t%s\\t%s\",\n                             insn.address, text_bytes, insn.mnemonic, insn.op_str)\n            raise InstructionEmulationError(str(e))", "response": "Emulate the given instruction object."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef backup_emulate(self, insn):\n\n        if not hasattr(self, 'backup_emu'):\n            self.backup_emu = UnicornEmulator(self)\n        try:\n            self.backup_emu.emulate(insn)\n        except unicorn.UcError as e:\n            if e.errno == unicorn.UC_ERR_INSN_INVALID:\n                text_bytes = ' '.join('%02x' % x for x in insn.bytes)\n                logger.error(\"Unimplemented instruction: 0x%016x:\\t%s\\t%s\\t%s\",\n                             insn.address, text_bytes, insn.mnemonic, insn.op_str)\n            raise InstructionEmulationError(str(e))\n        finally:\n            # We have been seeing occasional Unicorn issues with it not clearing\n            # the backing unicorn instance. Saw fewer issues with the following\n            # line present.\n            del self.backup_emu", "response": "Emulate an instruction and return the backup unicorn instance."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nhighlight the basic blocks of a Manticore trace file.", "response": "def viz_trace(view):\n    \"\"\"\n    Given a Manticore trace file, highlight the basic blocks.\n    \"\"\"\n    tv = TraceVisualizer(view, None)\n    if tv.workspace is None:\n        tv.workspace = get_workspace()\n    tv.visualize()"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef viz_live_trace(view):\n    tv = TraceVisualizer(view, None, live=True)\n    if tv.workspace is None:\n        tv.workspace = get_workspace()\n    # update due to singleton in case we are called after a clear\n    tv.live_update = True\n    tv.visualize()", "response": "Given a Manticore trace file highlight the basic blocks."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nhighlighting basic blocks from the Manticore workspace or trace file.", "response": "def visualize(self):\n        \"\"\"\n        Given a Manticore workspace, or trace file, highlight the basic blocks.\n        \"\"\"\n        if os.path.isfile(self.workspace):\n            t = threading.Thread(target=self.highlight_from_file,\n                                 args=(self.workspace,))\n        elif os.path.isdir(self.workspace):\n            t = threading.Thread(target=self.highlight_from_dir,\n                                 args=(self.workspace,))\n        t.start()"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nparses a single token and return the next token.", "response": "def t_TOKEN(t):\n    '[a-zA-Z0-9]+'\n    #print t.value,t.lexer.lexdata[t.lexer.lexpos-len(t.value):],re_TYPE.match(t.lexer.lexdata,t.lexer.lexpos-len(t.value))\n    if re_TYPE.match(t.value):\n        t.type = 'TYPE'\n    elif re_PTR.match(t.value):\n        t.type = 'PTR'\n    elif re_NUMBER.match(t.value):\n        if t.value.startswith('0x'):\n            t.value = t.value[2:]\n        t.value = int(t.value, 16)\n        t.type = 'NUMBER'\n    elif re_REGISTER.match(t.value):\n        t.type = 'REGISTER'\n    elif re_SEGMENT.match(t.value):\n        t.type = 'SEGMENT'\n    else:\n        raise Exception(f\"Unknown:<{t.value}>\")\n    return t"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef p_expression_deref(p):\n    'expression : TYPE PTR LBRAKET expression RBRAKET'\n    size = sizes[p[1]]\n    address = p[4]\n    char_list = functions['read_memory'](address, size)\n    value = Operators.CONCAT(8 * len(char_list), *reversed(map(Operators.ORD, char_list)))\n    p[0] = value", "response": "EXPRESSION TYPE PTR LBRAKET expression RBRAKET"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef p_expression_derefseg(p):\n    'expression : TYPE PTR SEGMENT COLOM LBRAKET expression RBRAKET'\n    size = sizes[p[1]]\n    address = p[6]\n    seg = functions['read_register'](p[3])\n    base, limit, _ = functions['get_descriptor'](seg)\n    address = base + address\n    char_list = functions['read_memory'](address, size)\n    value = Operators.CONCAT(8 * len(char_list), *reversed(map(Operators.ORD, char_list)))\n    p[0] = value", "response": "expression : TYPE PTR SEGMENT COLOM LBRAKET expression RBRAKET"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nbuilds EFLAGS and RFLAGS from flags", "response": "def _get_flags(self, reg):\n        \"\"\" Build EFLAGS/RFLAGS from flags \"\"\"\n        def make_symbolic(flag_expr):\n            register_size = 32 if reg == 'EFLAGS' else 64\n            value, offset = flag_expr\n            return Operators.ITEBV(register_size, value,\n                                   BitVecConstant(register_size, 1 << offset),\n                                   BitVecConstant(register_size, 0))\n\n        flags = []\n        for flag, offset in self._flags.items():\n            flags.append((self._registers[flag], offset))\n\n        if any(issymbolic(flag) for flag, offset in flags):\n            res = reduce(operator.or_, map(make_symbolic, flags))\n        else:\n            res = 0\n            for flag, offset in flags:\n                res += flag << offset\n        return res"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset individual flags from an EFLAGS or RFLAGS value.", "response": "def _set_flags(self, reg, res):\n        \"\"\" Set individual flags from a EFLAGS/RFLAGS value \"\"\"\n        #assert sizeof (res) == 32 if reg == 'EFLAGS' else 64\n        for flag, offset in self._flags.items():\n            self.write(flag, Operators.EXTRACT(res, offset, 1))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef push(cpu, value, size):\n        assert size in (8, 16, cpu.address_bit_size)\n        cpu.STACK = cpu.STACK - size // 8\n        base, _, _ = cpu.get_descriptor(cpu.read_register('SS'))\n        address = cpu.STACK + base\n        cpu.write_int(address, value, size)", "response": "Writes a value into the stack."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pop(cpu, size):\n        assert size in (16, cpu.address_bit_size)\n        base, _, _ = cpu.get_descriptor(cpu.SS)\n        address = cpu.STACK + base\n        value = cpu.read_int(address, size)\n        cpu.STACK = cpu.STACK + size // 8\n        return value", "response": "Get a value from the stack."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef invalidate_cache(cpu, address, size):\n        cache = cpu.instruction_cache\n        for offset in range(size):\n            if address + offset in cache:\n                del cache[address + offset]", "response": "invalidates the instruction cache at the given address."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef AND(cpu, dest, src):\n        # XXX bypass a capstone bug that incorrectly extends and computes operands sizes\n        # the bug has been fixed since capstone 4.0.alpha2 (commit de8dd26)\n        if src.size == 64 and src.type == 'immediate' and dest.size == 64:\n            arg1 = Operators.SEXTEND(src.read(), 32, 64)\n        else:\n            arg1 = src.read()\n        res = dest.write(dest.read() & arg1)\n        # Defined Flags: szp\n        cpu._calculate_logic_flags(dest.size, res)", "response": "Logical AND operation on the destination and source sets the result in the destination operand location."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef XOR(cpu, dest, src):\n        if dest == src:\n            # if the operands are the same write zero\n            res = dest.write(0)\n        else:\n            res = dest.write(dest.read() ^ src.read())\n        # Defined Flags: szp\n        cpu._calculate_logic_flags(dest.size, res)", "response": "Logical OR operation on the destination and source sets the result in the destination and source locations."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef OR(cpu, dest, src):\n        res = dest.write(dest.read() | src.read())\n        # Defined Flags: szp\n        cpu._calculate_logic_flags(dest.size, res)", "response": "Logical inclusive OR.\n\n        Performs a bitwise inclusive OR operation between the destination (first)\n        and source (second) operands and stores the result in the destination operand location.\n\n        Each bit of the result of the OR instruction is set to 0 if both corresponding\n        bits of the first and second operands are 0; otherwise, each bit is set\n        to 1.\n\n        The OF and CF flags are cleared; the SF, ZF, and PF flags are set according to the result::\n\n            DEST  =  DEST OR SRC;\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: source operand."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef AAA(cpu):\n        cpu.AF = Operators.OR(cpu.AL & 0x0F > 9, cpu.AF)\n        cpu.CF = cpu.AF\n        cpu.AH = Operators.ITEBV(8, cpu.AF, cpu.AH + 1, cpu.AH)\n        cpu.AL = Operators.ITEBV(8, cpu.AF, cpu.AL + 6, cpu.AL)\n        \"\"\"\n        if (cpu.AL & 0x0F > 9) or cpu.AF == 1:\n            cpu.AL = cpu.AL + 6\n            cpu.AH = cpu.AH + 1\n            cpu.AF = True\n            cpu.CF = True\n        else:\n            cpu.AF = False\n            cpu.CF = False\n        \"\"\"\n        cpu.AL = cpu.AL & 0x0f", "response": "AAA instruction for handling the AAA instruction."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef AAD(cpu, imm=None):\n        if imm is None:\n            imm = 10\n        else:\n            imm = imm.read()\n\n        cpu.AL += cpu.AH * imm\n        cpu.AH = 0\n\n        # Defined flags: ...sz.p.\n        cpu._calculate_logic_flags(8, cpu.AL)", "response": "AAD instruction for determining the base 10 value of the current AH register."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef AAM(cpu, imm=None):\n        if imm is None:\n            imm = 10\n        else:\n            imm = imm.read()\n\n        cpu.AH = Operators.UDIV(cpu.AL, imm)\n        cpu.AL = Operators.UREM(cpu.AL, imm)\n\n        # Defined flags: ...sz.p.\n        cpu._calculate_logic_flags(8, cpu.AL)", "response": "AAM instruction for storing the result of an AAM instruction."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nadding with carry. Adds the destination operand (first operand), the source operand (second operand), and the carry (CF) flag and stores the result in the destination operand. The state of the CF flag represents a carry from a previous addition. When an immediate value is used as an operand, it is sign-extended to the length of the destination operand format. The ADC instruction does not distinguish between signed or unsigned operands. Instead, the processor evaluates the result for both data types and sets the OF and CF flags to indicate a carry in the signed or unsigned result, respectively. The SF flag indicates the sign of the signed result. The ADC instruction is usually executed as part of a multibyte or multiword addition in which an ADD instruction is followed by an ADC instruction:: DEST = DEST + SRC + CF; The OF, SF, ZF, AF, CF, and PF flags are set according to the result. :param cpu: current CPU. :param dest: destination operand. :param src: source operand.", "response": "def ADC(cpu, dest, src):\n        \"\"\"\n        Adds with carry.\n\n        Adds the destination operand (first operand), the source operand (second operand),\n        and the carry (CF) flag and stores the result in the destination operand. The state\n        of the CF flag represents a carry from a previous addition. When an immediate value\n        is used as an operand, it is sign-extended to the length of the destination operand\n        format. The ADC instruction does not distinguish between signed or unsigned operands.\n        Instead, the processor evaluates the result for both data types and sets the OF and CF\n        flags to indicate a carry in the signed or unsigned result, respectively. The SF flag\n        indicates the sign of the signed result. The ADC instruction is usually executed as\n        part of a multibyte or multiword addition in which an ADD instruction is followed by an\n        ADC instruction::\n\n                DEST  =  DEST + SRC + CF;\n\n        The OF, SF, ZF, AF, CF, and PF flags are set according to the result.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: source operand.\n        \"\"\"\n        cpu._ADD(dest, src, carry=True)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ADD(cpu, dest, src):\n        cpu._ADD(dest, src, carry=False)", "response": "A base implementation of the ADD instruction."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CMP(cpu, src1, src2):\n        arg0 = src1.read()\n        arg1 = Operators.SEXTEND(src2.read(), src2.size, src1.size)\n\n        # Affected Flags o..szapc\n        cpu._calculate_CMP_flags(src1.size, arg0 - arg1, arg0, arg1)", "response": "Compares two source and destination sets the status flags of the current entry in the EFLAGS register according to the result of the comparison."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CMPXCHG(cpu, dest, src):\n        size = dest.size\n        reg_name = {8: 'AL', 16: 'AX', 32: 'EAX', 64: 'RAX'}[size]\n        accumulator = cpu.read_register(reg_name)\n        sval = src.read()\n        dval = dest.read()\n\n        cpu.write_register(reg_name, dval)\n        dest.write(Operators.ITEBV(size, accumulator == dval, sval, dval))\n\n        # Affected Flags o..szapc\n        cpu._calculate_CMP_flags(size, accumulator - dval, accumulator, dval)", "response": "This function is used to compare two sets of entries in the current memory with the source memory."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncompare and exchanges bytes. Compares the 64-bit value in EDX:EAX (or 128-bit value in RDX:RAX if operand size is 128 bits) with the operand (destination operand). If the values are equal, the 64-bit value in ECX:EBX (or 128-bit value in RCX:RBX) is stored in the destination operand. Otherwise, the value in the destination operand is loaded into EDX:EAX (or RDX:RAX):: IF (64-Bit Mode and OperandSize = 64) THEN IF (RDX:RAX = DEST) THEN ZF = 1; DEST = RCX:RBX; ELSE ZF = 0; RDX:RAX = DEST; FI ELSE IF (EDX:EAX = DEST) THEN ZF = 1; DEST = ECX:EBX; ELSE ZF = 0; EDX:EAX = DEST; FI; FI; :param cpu: current CPU. :param dest: destination operand.", "response": "def CMPXCHG8B(cpu, dest):\n        \"\"\"\n        Compares and exchanges bytes.\n\n        Compares the 64-bit value in EDX:EAX (or 128-bit value in RDX:RAX if\n        operand size is 128 bits) with the operand (destination operand). If\n        the values are equal, the 64-bit value in ECX:EBX (or 128-bit value in\n        RCX:RBX) is stored in the destination operand.  Otherwise, the value in\n        the destination operand is loaded into EDX:EAX (or RDX:RAX)::\n\n                IF (64-Bit Mode and OperandSize = 64)\n                THEN\n                    IF (RDX:RAX = DEST)\n                    THEN\n                        ZF = 1;\n                        DEST = RCX:RBX;\n                    ELSE\n                        ZF = 0;\n                        RDX:RAX = DEST;\n                    FI\n                ELSE\n                    IF (EDX:EAX = DEST)\n                    THEN\n                        ZF = 1;\n                        DEST = ECX:EBX;\n                    ELSE\n                        ZF = 0;\n                        EDX:EAX = DEST;\n                    FI;\n                FI;\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        \"\"\"\n        size = dest.size\n        cmp_reg_name_l = {64: 'EAX', 128: 'RAX'}[size]\n        cmp_reg_name_h = {64: 'EDX', 128: 'RDX'}[size]\n        src_reg_name_l = {64: 'EBX', 128: 'RBX'}[size]\n        src_reg_name_h = {64: 'ECX', 128: 'RCX'}[size]\n\n        # EDX:EAX or RDX:RAX\n        cmph = cpu.read_register(cmp_reg_name_h)\n        cmpl = cpu.read_register(cmp_reg_name_l)\n\n        srch = cpu.read_register(src_reg_name_h)\n        srcl = cpu.read_register(src_reg_name_l)\n\n        cmp0 = Operators.CONCAT(size, cmph, cmpl)\n        src0 = Operators.CONCAT(size, srch, srcl)\n        arg_dest = dest.read()\n        cpu.ZF = arg_dest == cmp0\n\n        dest.write(\n            Operators.ITEBV(size, cpu.ZF,\n                            Operators.CONCAT(size, srch, srcl),\n                            arg_dest)\n        )\n        cpu.write_register(cmp_reg_name_l, Operators.ITEBV(size // 2, cpu.ZF, cmpl,\n                                                           Operators.EXTRACT(arg_dest, 0, size // 2)))\n        cpu.write_register(cmp_reg_name_h, Operators.ITEBV(size // 2, cpu.ZF, cmph,\n                                                           Operators.EXTRACT(arg_dest, size // 2, size // 2)))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef DAS(cpu):\n        oldAL = cpu.AL\n        oldCF = cpu.CF\n\n        cpu.AF = Operators.OR((cpu.AL & 0x0f) > 9, cpu.AF)\n        cpu.AL = Operators.ITEBV(8, cpu.AF, cpu.AL - 6, cpu.AL)\n        cpu.CF = Operators.ITE(cpu.AF, Operators.OR(oldCF, cpu.AL > oldAL), cpu.CF)\n\n        cpu.CF = Operators.ITE(Operators.OR(oldAL > 0x99, oldCF), True, cpu.CF)\n        cpu.AL = Operators.ITEBV(8, Operators.OR(oldAL > 0x99, oldCF), cpu.AL - 0x60, cpu.AL)\n        #\n        \"\"\"\n        if (cpu.AL & 0x0f) > 9 or cpu.AF:\n            cpu.AL = cpu.AL - 6;\n            cpu.CF = Operators.OR(oldCF, cpu.AL > oldAL)\n            cpu.AF = True\n        else:\n            cpu.AF  =  False\n\n        if ((oldAL > 0x99) or oldCF):\n            cpu.AL = cpu.AL - 0x60\n            cpu.CF = True\n        \"\"\"\n        cpu.ZF = cpu.AL == 0\n        cpu.SF = (cpu.AL & 0x80) != 0\n        cpu.PF = cpu._calculate_parity_flag(cpu.AL)", "response": "This function is used to create a packed BCD result for a specific CPU."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef DIV(cpu, src):\n        size = src.size\n        reg_name_h = {8: 'DL', 16: 'DX', 32: 'EDX', 64: 'RDX'}[size]\n        reg_name_l = {8: 'AL', 16: 'AX', 32: 'EAX', 64: 'RAX'}[size]\n\n        dividend = Operators.CONCAT(size * 2,\n                                    cpu.read_register(reg_name_h),\n                                    cpu.read_register(reg_name_l))\n        divisor = Operators.ZEXTEND(src.read(), size * 2)\n\n        # TODO make symbol friendly\n        if isinstance(divisor, int) and divisor == 0:\n            raise DivideByZeroError()\n        quotient = Operators.UDIV(dividend, divisor)\n\n        MASK = (1 << size) - 1\n\n        # TODO make symbol friendly\n        if isinstance(quotient, int) and quotient > MASK:\n            raise DivideByZeroError()\n        remainder = Operators.UREM(dividend, divisor)\n\n        cpu.write_register(reg_name_l, Operators.EXTRACT(quotient, 0, size))\n        cpu.write_register(reg_name_h, Operators.EXTRACT(remainder, 0, size))", "response": "This function divide the value in the instruction and stores the result in the instruction instruction. instruction instruction. instruction is used to determine the value of the instruction instruction. instruction."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef IDIV(cpu, src):\n\n        reg_name_h = {8: 'AH', 16: 'DX', 32: 'EDX', 64: 'RDX'}[src.size]\n        reg_name_l = {8: 'AL', 16: 'AX', 32: 'EAX', 64: 'RAX'}[src.size]\n\n        dividend = Operators.CONCAT(src.size * 2,\n                                    cpu.read_register(reg_name_h),\n                                    cpu.read_register(reg_name_l))\n\n        divisor = src.read()\n        if isinstance(divisor, int) and divisor == 0:\n            raise DivideByZeroError()\n\n        dst_size = src.size * 2\n\n        divisor = Operators.SEXTEND(divisor, src.size, dst_size)\n        mask = (1 << dst_size) - 1\n        sign_mask = 1 << (dst_size - 1)\n\n        dividend_sign = (dividend & sign_mask) != 0\n        divisor_sign = (divisor & sign_mask) != 0\n\n        if isinstance(divisor, int):\n            if divisor_sign:\n                divisor = ((~divisor) + 1) & mask\n                divisor = -divisor\n\n        if isinstance(dividend, int):\n            if dividend_sign:\n                dividend = ((~dividend) + 1) & mask\n                dividend = -dividend\n\n        quotient = Operators.SDIV(dividend, divisor)\n        if (isinstance(dividend, int) and\n                isinstance(dividend, int)):\n            # handle the concrete case\n            remainder = dividend - (quotient * divisor)\n        else:\n            # symbolic case -- optimize via SREM\n            remainder = Operators.SREM(dividend, divisor)\n\n        cpu.write_register(reg_name_l, Operators.EXTRACT(quotient, 0, src.size))\n        cpu.write_register(reg_name_h, Operators.EXTRACT(remainder, 0, src.size))", "response": "This function is used to divide the value of the source and store it in the AL AX or EAX registers. The value of the source is divided by the value of the source and stores the result in the EAX DX and EDX registers."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef IMUL(cpu, *operands):\n        dest = operands[0]\n        OperandSize = dest.size\n        reg_name_h = {8: 'AH', 16: 'DX', 32: 'EDX', 64: 'RDX'}[OperandSize]\n        reg_name_l = {8: 'AL', 16: 'AX', 32: 'EAX', 64: 'RAX'}[OperandSize]\n\n        arg0 = dest.read()\n        arg1 = None\n        arg2 = None\n        res = None\n        if len(operands) == 1:\n            arg1 = cpu.read_register(reg_name_l)\n            temp = (Operators.SEXTEND(arg0, OperandSize, OperandSize * 2) *\n                    Operators.SEXTEND(arg1, OperandSize, OperandSize * 2))\n            temp = temp & ((1 << (OperandSize * 2)) - 1)\n            cpu.write_register(reg_name_l,\n                               Operators.EXTRACT(temp, 0, OperandSize))\n            cpu.write_register(reg_name_h,\n                               Operators.EXTRACT(temp, OperandSize, OperandSize))\n            res = Operators.EXTRACT(temp, 0, OperandSize)\n        elif len(operands) == 2:\n            arg1 = operands[1].read()\n            arg1 = Operators.SEXTEND(arg1, OperandSize, OperandSize * 2)\n            temp = Operators.SEXTEND(arg0, OperandSize, OperandSize * 2) * arg1\n            temp = temp & ((1 << (OperandSize * 2)) - 1)\n            res = dest.write(Operators.EXTRACT(temp, 0, OperandSize))\n        else:\n            arg1 = operands[1].read()\n            arg2 = operands[2].read()\n            temp = (Operators.SEXTEND(arg1, OperandSize, OperandSize * 2) *\n                    Operators.SEXTEND(arg2, operands[2].size, OperandSize * 2))\n            temp = temp & ((1 << (OperandSize * 2)) - 1)\n            res = dest.write(Operators.EXTRACT(temp, 0, OperandSize))\n\n        cpu.CF = (Operators.SEXTEND(res, OperandSize, OperandSize * 2) != temp)\n        cpu.OF = cpu.CF", "response": "This function performs a signed multiplication of two values in the specified CPU."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef INC(cpu, dest):\n        arg0 = dest.read()\n        res = dest.write(arg0 + 1)\n        res &= (1 << dest.size) - 1\n        SIGN_MASK = 1 << (dest.size - 1)\n        cpu.AF = ((arg0 ^ 1) ^ res) & 0x10 != 0\n        cpu.ZF = res == 0\n        cpu.SF = (res & SIGN_MASK) != 0\n        cpu.OF = res == SIGN_MASK\n        cpu.PF = cpu._calculate_parity_flag(res)", "response": "Increments by 1. The destination operand is a register or memory location."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef SBB(cpu, dest, src):\n        cpu._SUB(dest, src, carry=True)", "response": "Integer subtraction with borrow."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef SUB(cpu, dest, src):\n        cpu._SUB(dest, src, carry=False)", "response": "Subtract.\n\n        Subtracts the second operand (source operand) from the first operand\n        (destination operand) and stores the result in the destination operand.\n        The destination operand can be a register or a memory location; the\n        source operand can be an immediate, register, or memory location.\n        (However, two memory operands cannot be used in one instruction.) When\n        an immediate value is used as an operand, it is sign-extended to the\n        length of the destination operand format.\n        The SUB instruction does not distinguish between signed or unsigned\n        operands. Instead, the processor evaluates the result for both\n        data types and sets the OF and CF flags to indicate a borrow in the\n        signed or unsigned result, respectively. The SF flag indicates the sign\n        of the signed result::\n\n            DEST  =  DEST - SRC;\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: source operand."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nexchanges and adds. Exchanges the first operand (destination operand) with the second operand (source operand), then loads the sum of the two values into the destination operand. The destination operand can be a register or a memory location; the source operand is a register. This instruction can be used with a LOCK prefix:: TEMP = SRC + DEST SRC = DEST DEST = TEMP :param cpu: current CPU. :param dest: destination operand. :param src: source operand.", "response": "def XADD(cpu, dest, src):\n        \"\"\"\n        Exchanges and adds.\n\n        Exchanges the first operand (destination operand) with the second operand\n        (source operand), then loads the sum of the two values into the destination\n        operand. The destination operand can be a register or a memory location;\n        the source operand is a register.\n        This instruction can be used with a LOCK prefix::\n\n                TEMP  =  SRC + DEST\n                SRC  =  DEST\n                DEST  =  TEMP\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: source operand.\n        \"\"\"\n        MASK = (1 << dest.size) - 1\n        SIGN_MASK = 1 << (dest.size - 1)\n\n        arg0 = dest.read()\n        arg1 = src.read()\n        temp = (arg1 + arg0) & MASK\n        src.write(arg0)\n        dest.write(temp)\n\n        # Affected flags: oszapc\n        tempCF = Operators.OR(Operators.ULT(temp, arg0), Operators.ULT(temp, arg1))\n        cpu.CF = tempCF\n        cpu.AF = ((arg0 ^ arg1) ^ temp) & 0x10 != 0\n        cpu.ZF = temp == 0\n        cpu.SF = (temp & SIGN_MASK) != 0\n        cpu.OF = (((arg0 ^ arg1 ^ SIGN_MASK) & (temp ^ arg1)) & SIGN_MASK) != 0\n        cpu.PF = cpu._calculate_parity_flag(temp)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef BSWAP(cpu, dest):\n        parts = []\n        arg0 = dest.read()\n        for i in range(0, dest.size, 8):\n            parts.append(Operators.EXTRACT(arg0, i, 8))\n\n        dest.write(Operators.CONCAT(8 * len(parts), *parts))", "response": "Byte swap.\n\n        Reverses the byte order of a 32-bit (destination) register: bits 0 through\n        7 are swapped with bits 24 through 31, and bits 8 through 15 are swapped\n        with bits 16 through 23. This instruction is provided for converting little-endian\n        values to big-endian format and vice versa.\n        To swap bytes in a word value (16-bit register), use the XCHG instruction.\n        When the BSWAP instruction references a 16-bit register, the result is\n        undefined::\n\n            TEMP  =  DEST\n            DEST[7..0]  =  TEMP[31..24]\n            DEST[15..8]  =  TEMP[23..16]\n            DEST[23..16]  =  TEMP[15..8]\n            DEST[31..24]  =  TEMP[7..0]\n\n        :param cpu: current CPU.\n        :param dest: destination operand."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CMOVAE(cpu, dest, src):\n        dest.write(Operators.ITEBV(dest.size, cpu.CF == False, src.read(), dest.read()))", "response": "Conditional move - Above or not below."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CMOVZ(cpu, dest, src):\n        dest.write(Operators.ITEBV(dest.size, cpu.ZF, src.read(), dest.read()))", "response": "Conditional move - Equal or zero."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef CMOVP(cpu, dest, src):\n        dest.write(Operators.ITEBV(dest.size, cpu.PF, src.read(), dest.read()))", "response": "Conditional move - Parity even."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CMOVNP(cpu, dest, src):\n        dest.write(Operators.ITEBV(dest.size, cpu.PF == False, src.read(), dest.read()))", "response": "Conditional move - Not parity odd."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef CMOVNO(cpu, dest, src):\n        dest.write(Operators.ITEBV(dest.size, cpu.OF == False, src.read(), dest.read()))", "response": "Conditional move - Not overflow."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef CMOVNS(cpu, dest, src):\n        dest.write(Operators.ITEBV(dest.size, cpu.SF == False, src.read(), dest.read()))", "response": "Conditional move - Not sign ( non - negative."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nloading the status flags into the AH register.", "response": "def LAHF(cpu):\n        \"\"\"\n        Loads status flags into AH register.\n\n        Moves the low byte of the EFLAGS register (which includes status flags\n        SF, ZF, AF, PF, and CF) to the AH register. Reserved bits 1, 3, and 5\n        of the EFLAGS register are set in the AH register::\n\n                AH  =  EFLAGS(SF:ZF:0:AF:0:PF:1:CF);\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: source operand.\n        \"\"\"\n        used_regs = (cpu.SF, cpu.ZF, cpu.AF, cpu.PF, cpu.CF)\n        is_expression = any(issymbolic(x) for x in used_regs)\n\n        def make_flag(val, offset):\n            if is_expression:\n                return Operators.ITEBV(8, val,\n                                       BitVecConstant(8, 1 << offset),\n                                       BitVecConstant(8, 0))\n            else:\n                return val << offset\n\n        cpu.AH = (make_flag(cpu.SF, 7) |\n                  make_flag(cpu.ZF, 6) |\n                  make_flag(0, 5) |\n                  make_flag(cpu.AF, 4) |\n                  make_flag(0, 3) |\n                  make_flag(cpu.PF, 2) |\n                  make_flag(1, 1) |\n                  make_flag(cpu.CF, 0))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nloading the effective address of the source operand and store it in the destination operand.", "response": "def LEA(cpu, dest, src):\n        \"\"\"\n        Loads effective address.\n\n        Computes the effective address of the second operand (the source operand) and stores it in the first operand\n        (destination operand). The source operand is a memory address (offset part) specified with one of the processors\n        addressing modes; the destination operand is a general-purpose register. The address-size and operand-size\n        attributes affect the action performed by this instruction. The operand-size\n        attribute of the instruction is determined by the chosen register; the address-size attribute is determined by the\n        attribute of the code segment.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: source operand.\n        \"\"\"\n        dest.write(Operators.EXTRACT(src.address(), 0, dest.size))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef MOVBE(cpu, dest, src):\n        size = dest.size\n        arg0 = dest.read()\n        temp = 0\n        for pos in range(0, size, 8):\n            temp = (temp << 8) | (arg0 & 0xff)\n            arg0 = arg0 >> 8\n        dest.write(arg0)", "response": "This function moves data after swapping bytes."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef SAHF(cpu):\n\n        eflags_size = 32\n        val = cpu.AH & 0xD5 | 0x02\n\n        cpu.EFLAGS = Operators.ZEXTEND(val, eflags_size)", "response": "Stores AH into flags."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef SETA(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, Operators.OR(cpu.CF, cpu.ZF) == False, 1, 0))", "response": "Sets the destination operand to 0 or 1 depending on the status flags of the current CPU."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef SETB(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, cpu.CF, 1, 0))", "response": "Sets byte if below."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset byte if below or equal.", "response": "def SETBE(cpu, dest):\n        \"\"\"\n        Sets byte if below or equal.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        \"\"\"\n        dest.write(Operators.ITEBV(dest.size, Operators.OR(cpu.CF, cpu.ZF), 1, 0))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset if carry. :param cpu: current CPU. :param dest: destination operand.", "response": "def SETC(cpu, dest):\n        \"\"\"\n        Sets if carry.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        \"\"\"\n        dest.write(Operators.ITEBV(dest.size, cpu.CF, 1, 0))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset byte if equal.", "response": "def SETE(cpu, dest):\n        \"\"\"\n        Sets byte if equal.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        \"\"\"\n        dest.write(Operators.ITEBV(dest.size, cpu.ZF, 1, 0))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef SETGE(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, cpu.SF == cpu.OF, 1, 0))", "response": "Sets byte if greater or equal."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting byte if not above or equal.", "response": "def SETNAE(cpu, dest):\n        \"\"\"\n        Sets byte if not above or equal.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        \"\"\"\n        dest.write(Operators.ITEBV(dest.size, cpu.CF, 1, 0))"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets byte if not below.", "response": "def SETNB(cpu, dest):\n        \"\"\"\n        Sets byte if not below.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        \"\"\"\n        dest.write(Operators.ITEBV(dest.size, cpu.CF == False, 1, 0))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef SETNBE(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, Operators.AND(cpu.CF == False, cpu.ZF == False), 1, 0))", "response": "Sets byte if not below or equal."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef SETNG(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, Operators.OR(cpu.ZF, cpu.SF != cpu.OF), 1, 0))", "response": "Sets byte if not greater."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef SETNLE(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, Operators.AND(cpu.ZF == False, cpu.SF == cpu.OF), 1, 0))", "response": "Sets byte if not less or equal."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef SETNO(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, cpu.OF == False, 1, 0))", "response": "Sets byte if not overflow."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting byte if not sign.", "response": "def SETNS(cpu, dest):\n        \"\"\"\n        Sets byte if not sign.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        \"\"\"\n        dest.write(Operators.ITEBV(dest.size, cpu.SF == False, 1, 0))"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting byte if not zero.", "response": "def SETNZ(cpu, dest):\n        \"\"\"\n        Sets byte if not zero.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        \"\"\"\n        dest.write(Operators.ITEBV(dest.size, cpu.ZF == False, 1, 0))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef SETO(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, cpu.OF, 1, 0))", "response": "Sets byte if overflow."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef SETP(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, cpu.PF, 1, 0))", "response": "Sets byte if parity."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef SETPE(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, cpu.PF, 1, 0))", "response": "Sets byte if parity even."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset byte if parity odd.", "response": "def SETPO(cpu, dest):\n        \"\"\"\n        Sets byte if parity odd.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        \"\"\"\n        dest.write(Operators.ITEBV(dest.size, cpu.PF == False, 1, 0))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef SETS(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, cpu.SF, 1, 0))", "response": "Sets byte if sign.\n       "}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef SETZ(cpu, dest):\n        dest.write(Operators.ITEBV(dest.size, cpu.ZF, 1, 0))", "response": "Sets byte if zero."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef XCHG(cpu, dest, src):\n        temp = dest.read()\n        dest.write(src.read())\n        src.write(temp)", "response": "This instruction is used to set the contents of the destination and source memory locations for the specified CPU."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef LEAVE(cpu):\n        cpu.STACK = cpu.FRAME\n        cpu.FRAME = cpu.pop(cpu.address_bit_size)", "response": "This function is called by the ENTER instruction to release the stack frame allocated by the calling procedure s stack."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\npush a value onto the stack.", "response": "def PUSH(cpu, src):\n        \"\"\"\n        Pushes a value onto the stack.\n\n        Decrements the stack pointer and then stores the source operand on the top of the stack.\n\n        :param cpu: current CPU.\n        :param src: source operand.\n        \"\"\"\n        # http://stackoverflow.com/questions/11291151/how-push-imm-encodes\n        size = src.size\n        v = src.read()\n        if size != 64 and size != cpu.address_bit_size // 2:\n            v = Operators.SEXTEND(v, size, cpu.address_bit_size)\n            size = cpu.address_bit_size\n        cpu.push(v, size)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\npop stack into EFLAGS register.", "response": "def POPF(cpu):\n        \"\"\"\n        Pops stack into EFLAGS register.\n\n        :param cpu: current CPU.\n        \"\"\"\n        mask = (0x00000001 |\n                0x00000004 |\n                0x00000010 |\n                0x00000040 |\n                0x00000080 |\n                0x00000400 |\n                0x00000800)\n        val = cpu.pop(16)\n        eflags_size = 32\n        cpu.EFLAGS = Operators.ZEXTEND(val & mask, eflags_size)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\npop stack into EFLAGS register.", "response": "def POPFQ(cpu):\n        \"\"\"\n        Pops stack into EFLAGS register.\n\n        :param cpu: current CPU.\n        \"\"\"\n        mask = 0x00000001 | 0x00000004 | 0x00000010 | 0x00000040 | 0x00000080 | 0x00000400 | 0x00000800\n        cpu.EFLAGS = (cpu.EFLAGS & ~mask) | cpu.pop(64) & mask"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncall to interrupt procedure. The INT n instruction generates a call to the interrupt or exception handler specified with the destination operand. The INT n instruction is the general mnemonic for executing a software-generated call to an interrupt handler. The INTO instruction is a special mnemonic for calling overflow exception (#OF), interrupt vector number 4. The overflow interrupt checks the OF flag in the EFLAGS register and calls the overflow interrupt handler if the OF flag is set to 1. :param cpu: current CPU. :param op0: destination operand.", "response": "def INT(cpu, op0):\n        \"\"\"\n        Calls to interrupt procedure.\n\n        The INT n instruction generates a call to the interrupt or exception handler specified\n        with the destination operand. The INT n instruction is the  general mnemonic for executing\n        a software-generated call to an interrupt handler. The INTO instruction is a special\n        mnemonic for calling overflow exception (#OF), interrupt vector number 4. The overflow\n        interrupt checks the OF flag in the EFLAGS register and calls the overflow interrupt handler\n        if the OF flag is set to 1.\n\n        :param cpu: current CPU.\n        :param op0: destination operand.\n        \"\"\"\n        if op0.read() != 0x80:\n            logger.warning(\"Unsupported interrupt\")\n        raise Interruption(op0.read())"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\njump short if above.", "response": "def JA(cpu, target):\n        \"\"\"\n        Jumps short if above.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, Operators.AND(cpu.CF == False, cpu.ZF == False), target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef JB(cpu, target):\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, cpu.CF == True, target.read(), cpu.PC)", "response": "Jumps short if below."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef JBE(cpu, target):\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, Operators.OR(cpu.CF, cpu.ZF), target.read(), cpu.PC)", "response": "Jumps short if below or equal."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef JC(cpu, target):\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, cpu.CF, target.read(), cpu.PC)", "response": "Jumps short if carry."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\njump short if CX register is 0.", "response": "def JCXZ(cpu, target):\n        \"\"\"\n        Jumps short if CX register is 0.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, cpu.CX == 0, target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef JECXZ(cpu, target):\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, cpu.ECX == 0, target.read(), cpu.PC)", "response": "Jumps short if ECX register is 0."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\njumping short if RCX register is 0.", "response": "def JRCXZ(cpu, target):\n        \"\"\"\n        Jumps short if RCX register is 0.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, cpu.RCX == 0, target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\njumps short if greater.", "response": "def JG(cpu, target):\n        \"\"\"\n        Jumps short if greater.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, Operators.AND(cpu.ZF == False, cpu.SF == cpu.OF), target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\njump short if greater or equal.", "response": "def JGE(cpu, target):\n        \"\"\"\n        Jumps short if greater or equal.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, (cpu.SF == cpu.OF), target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef JNB(cpu, target):\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, cpu.CF == False, target.read(), cpu.PC)", "response": "Jumps short if not below."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\njumping short if not equal.", "response": "def JNE(cpu, target):\n        \"\"\"\n        Jumps short if not equal.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, False == cpu.ZF, target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\njump short if not greater than target.", "response": "def JNG(cpu, target):\n        \"\"\"\n        Jumps short if not greater.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, Operators.OR(cpu.ZF, cpu.SF != cpu.OF), target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\njump short if not overflow.", "response": "def JNO(cpu, target):\n        \"\"\"\n        Jumps short if not overflow.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, False == cpu.OF, target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef JNP(cpu, target):\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, False == cpu.PF, target.read(), cpu.PC)", "response": "Jumps short if not parity."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef JNS(cpu, target):\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, False == cpu.SF, target.read(), cpu.PC)", "response": "Jumps short if not sign."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\njumping short if overflow.", "response": "def JO(cpu, target):\n        \"\"\"\n        Jumps short if overflow.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, cpu.OF, target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\njump short if parity.", "response": "def JP(cpu, target):\n        \"\"\"\n        Jumps short if parity.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, cpu.PF, target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\njumping short if sign.", "response": "def JS(cpu, target):\n        \"\"\"\n        Jumps short if sign.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, cpu.SF, target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\njump short if zero.", "response": "def JZ(cpu, target):\n        \"\"\"\n        Jumps short if zero.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, cpu.ZF, target.read(), cpu.PC)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\njumping to the target CS and PC", "response": "def LJMP(cpu, cs_selector, target):\n        \"\"\"\n        We are just going to ignore the CS selector for now.\n        \"\"\"\n        logger.info(\"LJMP: Jumping to: %r:%r\", cs_selector.read(), target.read())\n        cpu.CS = cs_selector.read()\n        cpu.PC = target.read()"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nloops according to ECX counter. Performs a loop operation using the ECX or CX register as a counter. Each time the LOOP instruction is executed, the count register is decremented, then checked for 0. If the count is 0, the loop is terminated and program execution continues with the instruction following the LOOP instruction. If the count is not zero, a near jump is performed to the destination (target) operand, which is presumably the instruction at the beginning of the loop. If the address-size attribute is 32 bits, the ECX register is used as the count register; otherwise the CX register is used:: IF address_bit_size = 32 THEN Count is ECX; ELSE (* address_bit_size = 16 *) Count is CX; FI; Count = Count - 1; IF (Count 0) = 1 THEN EIP = EIP + SignExtend(DEST); IF OperandSize = 16 THEN EIP = EIP AND 0000FFFFH; FI; ELSE Terminate loop and continue program execution at EIP; FI; :param cpu: current CPU. :param dest: destination operand.", "response": "def LOOP(cpu, dest):\n        \"\"\"\n        Loops according to ECX counter.\n\n        Performs a loop operation using the ECX or CX register as a counter.\n        Each time the LOOP instruction is executed, the count register is decremented,\n        then checked for 0. If the count is 0, the loop is terminated and program\n        execution continues with the instruction following the LOOP instruction.\n        If the count is not zero, a near jump is performed to the destination\n        (target) operand, which is presumably the instruction at the beginning\n        of the loop. If the address-size attribute is 32 bits, the ECX register\n        is used as the count register; otherwise the CX register is used::\n\n                IF address_bit_size  =  32\n                THEN\n                    Count is ECX;\n                ELSE (* address_bit_size  =  16 *)\n                    Count is CX;\n                FI;\n                Count  =  Count - 1;\n\n                IF (Count  0)  =  1\n                THEN\n                    EIP  =  EIP + SignExtend(DEST);\n                    IF OperandSize  =  16\n                    THEN\n                        EIP  =  EIP AND 0000FFFFH;\n                    FI;\n                ELSE\n                    Terminate loop and continue program execution at EIP;\n                FI;\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        \"\"\"\n        counter_name = {16: 'CX', 32: 'ECX', 64: 'RCX'}[cpu.address_bit_size]\n        counter = cpu.write_register(counter_name, cpu.read_register(counter_name) - 1)\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, counter == 0, (cpu.PC + dest.read()) & ((1 << dest.size) - 1), cpu.PC + cpu.instruction.size)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nloop if ECX counter is nonzero.", "response": "def LOOPNZ(cpu, target):\n        \"\"\"\n        Loops if ECX counter is nonzero.\n\n        :param cpu: current CPU.\n        :param target: destination operand.\n        \"\"\"\n        counter_name = {16: 'CX', 32: 'ECX', 64: 'RCX'}[cpu.address_bit_size]\n        counter = cpu.write_register(counter_name, cpu.read_register(counter_name) - 1)\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size, counter != 0, (cpu.PC + target.read()) & ((1 << target.size) - 1), cpu.PC + cpu.instruction.size)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrotate through carry left. Shifts (rotates) the bits of the first operand (destination operand) the number of bit positions specified in the second operand (count operand) and stores the result in the destination operand. The destination operand can be a register or a memory location; the count operand is an unsigned integer that can be an immediate or a value in the CL register. In legacy and compatibility mode, the processor restricts the count to a number between 0 and 31 by masking all the bits in the count operand except the 5 least-significant bits. The RCL instruction shifts the CF flag into the least-significant bit and shifts the most-significant bit into the CF flag. :param cpu: current CPU. :param dest: destination operand. :param src: count operand.", "response": "def RCL(cpu, dest, src):\n        \"\"\"\n        Rotates through carry left.\n\n        Shifts (rotates) the bits of the first operand (destination operand) the number of bit positions specified in the\n        second operand (count operand) and stores the result in the destination operand. The destination operand can be\n        a register or a memory location; the count operand is an unsigned integer that can be an immediate or a value in\n        the CL register. In legacy and compatibility mode, the processor restricts the count to a number between 0 and 31\n        by masking all the bits in the count operand except the 5 least-significant bits.\n\n        The RCL instruction shifts the CF flag into the least-significant bit and shifts the most-significant bit into the CF flag.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: count operand.\n        \"\"\"\n        OperandSize = dest.size\n        count = src.read()\n        countMask = {8: 0x1f,\n                     16: 0x1f,\n                     32: 0x1f,\n                     64: 0x3f}[OperandSize]\n        tempCount = Operators.ZEXTEND((count & countMask) % (src.size + 1), OperandSize)\n\n        value = dest.read()\n\n        if isinstance(tempCount, int) and tempCount == 0:\n            # this is a no-op\n            new_val = value\n            dest.write(new_val)\n        else:\n            carry = Operators.ITEBV(OperandSize, cpu.CF, 1, 0)\n            right = value >> (OperandSize - tempCount)\n            new_val = (value << tempCount) | (carry << (tempCount - 1)) | (right >> 1)\n            dest.write(new_val)\n\n            def sf(v, size):\n                return (v & (1 << (size - 1))) != 0\n            cpu.CF = sf(value << (tempCount - 1), OperandSize)\n            cpu.OF = Operators.ITE(tempCount == 1,\n                                   sf(new_val, OperandSize) != cpu.CF,\n                                   cpu.OF)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nrotating through carry right instruction and store the result in dest.", "response": "def RCR(cpu, dest, src):\n        \"\"\"\n        Rotates through carry right (RCR).\n\n        Shifts (rotates) the bits of the first operand (destination operand) the number of bit positions specified in the\n        second operand (count operand) and stores the result in the destination operand. The destination operand can be\n        a register or a memory location; the count operand is an unsigned integer that can be an immediate or a value in\n        the CL register. In legacy and compatibility mode, the processor restricts the count to a number between 0 and 31\n        by masking all the bits in the count operand except the 5 least-significant bits.\n\n        Rotate through carry right (RCR) instructions shift all the bits toward less significant bit positions, except\n        for the least-significant bit, which is rotated to the most-significant bit location. The RCR instruction shifts the\n        CF flag into the most-significant bit and shifts the least-significant bit into the CF flag.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: count operand.\n        \"\"\"\n        OperandSize = dest.size\n        count = src.read()\n        countMask = {8: 0x1f,\n                     16: 0x1f,\n                     32: 0x1f,\n                     64: 0x3f}[OperandSize]\n        tempCount = Operators.ZEXTEND((count & countMask) % (src.size + 1), OperandSize)\n\n        value = dest.read()\n        if isinstance(tempCount, int) and tempCount == 0:\n            # this is a no-op\n            new_val = value\n            dest.write(new_val)\n        else:\n            carry = Operators.ITEBV(OperandSize, cpu.CF, 1, 0)\n            left = value >> (tempCount - 1)\n            right = value << (OperandSize - tempCount)\n\n            new_val = (left >> 1) | (carry << (OperandSize - tempCount)) | (right << 1)\n\n            dest.write(new_val)\n\n            cpu.CF = Operators.ITE(tempCount != 0, (left & 1) == 1, cpu.CF)\n            # for RCR these are calculated before rotation starts\n            s_MSB = ((new_val >> (OperandSize - 1)) & 0x1) == 1\n            s_MSB2 = ((new_val >> (OperandSize - 2)) & 0x1) == 1\n            cpu.OF = Operators.ITE(tempCount == 1,\n                                   s_MSB ^ s_MSB2, cpu.OF)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrotate left shift the bits of the first operand and store the result in the destination operand.", "response": "def ROL(cpu, dest, src):\n        \"\"\"\n        Rotates left (ROL).\n\n        Shifts (rotates) the bits of the first operand (destination operand) the number of bit positions specified in the\n        second operand (count operand) and stores the result in the destination operand. The destination operand can be\n        a register or a memory location; the count operand is an unsigned integer that can be an immediate or a value in\n        the CL register. In legacy and compatibility mode, the processor restricts the count to a number between 0 and 31\n        by masking all the bits in the count operand except the 5 least-significant bits.\n\n        The rotate left shift all the bits toward more-significant bit positions, except for the most-significant bit, which\n        is rotated to the least-significant bit location.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: count operand.\n        \"\"\"\n        OperandSize = dest.size\n        count = src.read()\n        countMask = {8: 0x1f,\n                     16: 0x1f,\n                     32: 0x1f,\n                     64: 0x3f}[OperandSize]\n        tempCount = Operators.ZEXTEND((count & countMask) % (OperandSize), OperandSize)\n\n        value = dest.read()\n        newValue = (value << tempCount) | (value >> (OperandSize - tempCount))\n        dest.write(newValue)\n\n        cpu.CF = Operators.ITE(tempCount != 0, (newValue & 1) == 1, cpu.CF)\n        s_MSB = ((newValue >> (OperandSize - 1)) & 0x1) == 1\n        cpu.OF = Operators.ITE(tempCount == 1, s_MSB ^ cpu.CF, cpu.OF)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nshifts arithmetic right and shift logical right.", "response": "def SAR(cpu, dest, src):\n        \"\"\"\n        Shift arithmetic right.\n\n        The shift arithmetic right (SAR) and shift logical right (SHR) instructions shift the bits of the destination operand to\n        the right (toward less significant bit locations). For each shift count, the least significant bit of the destination\n        operand is shifted into the CF flag, and the most significant bit is either set or cleared depending on the instruction\n        type. The SHR instruction clears the most significant bit. the SAR instruction sets or clears the most significant bit\n        to correspond to the sign (most significant bit) of the original value in the destination operand. In effect, the SAR\n        instruction fills the empty bit position's shifted value with the sign of the unshifted value\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: source operand.\n        \"\"\"\n        OperandSize = dest.size\n        countMask = {8: 0x1f,\n                     16: 0x1f,\n                     32: 0x1f,\n                     64: 0x3f}[OperandSize]\n\n        count = src.read() & countMask\n        value = dest.read()\n\n        res = Operators.SAR(OperandSize, value, Operators.ZEXTEND(count, OperandSize))\n        dest.write(res)\n\n        SIGN_MASK = (1 << (OperandSize - 1))\n\n        # We can't use this one as the 'true' expression gets eagerly calculated even on count == 0\t\t +        cpu.CF = Operators.ITE(count!=0, ((value >> Operators.ZEXTEND(count-1, OperandSize)) & 1) !=0, cpu.CF)\n        # cpu.CF = Operators.ITE(count!=0, ((value >> Operators.ZEXTEND(count-1, OperandSize)) & 1) !=0, cpu.CF)\n\n        if issymbolic(count):\n            # We can't use this one as the EXTRACT op needs the offset arguments to be concrete\n            #    cpu.CF = Operators.ITE(count!=0, Operands.EXTRACT(value,count-1,1) !=0, cpu.CF)\n            cpu.CF = Operators.ITE(Operators.AND(count != 0, count <= OperandSize), ((value >> Operators.ZEXTEND(count - 1, OperandSize)) & 1) != 0, cpu.CF)\n        else:\n            if count != 0:\n                if count > OperandSize:\n                    count = OperandSize\n                cpu.CF = Operators.EXTRACT(value, count - 1, 1) != 0\n\n        # on count == 0 AF is unaffected, for count > 0, AF is undefined.\n        # in either case, do not touch AF\n        cpu.ZF = Operators.ITE(count != 0, res == 0, cpu.ZF)\n        cpu.SF = Operators.ITE(count != 0, (res & SIGN_MASK) != 0, cpu.SF)\n        cpu.OF = Operators.ITE(count == 1, False, cpu.OF)\n        cpu.PF = Operators.ITE(count != 0, cpu._calculate_parity_flag(res), cpu.PF)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nshift logical right instruction.", "response": "def SHR(cpu, dest, src):\n        \"\"\"\n        Shift logical right.\n\n        The shift arithmetic right (SAR) and shift logical right (SHR)\n        instructions shift the bits of the destination operand to the right\n        (toward less significant bit locations). For each shift count, the\n        least significant bit of the destination operand is shifted into the CF\n        flag, and the most significant bit is either set or cleared depending\n        on the instruction type. The SHR instruction clears the most\n        significant bit.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: count operand.\n        \"\"\"\n        OperandSize = dest.size\n        count = Operators.ZEXTEND(src.read() & (OperandSize - 1), OperandSize)\n        value = dest.read()\n\n        res = dest.write(value >> count)  # UNSIGNED Operators.UDIV2 !! TODO Check\n\n        MASK = (1 << OperandSize) - 1\n        SIGN_MASK = 1 << (OperandSize - 1)\n\n        if issymbolic(count):\n            cpu.CF = Operators.ITE(count != 0,\n                                   ((value >> Operators.ZEXTEND(count - 1, OperandSize)) & 1) != 0,\n                                   cpu.CF)\n        else:\n            if count != 0:\n                cpu.CF = Operators.EXTRACT(value, count - 1, 1) != 0\n\n        cpu.ZF = Operators.ITE(count != 0, res == 0, cpu.ZF)\n        cpu.SF = Operators.ITE(count != 0, (res & SIGN_MASK) != 0, cpu.SF)\n        # OF is only defined for count == 1, but in practice (unit tests from real cpu) it's calculated for count != 0\n        cpu.OF = Operators.ITE(count != 0, ((value >> (OperandSize - 1)) & 0x1) == 1, cpu.OF)\n        cpu.PF = Operators.ITE(count != 0, cpu._calculate_parity_flag(res), cpu.PF)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef SHLD(cpu, dest, src, count):\n        OperandSize = dest.size\n        tempCount = Operators.ZEXTEND(count.read(), OperandSize) & (OperandSize - 1)\n        arg0 = dest.read()\n        arg1 = src.read()\n\n        MASK = ((1 << OperandSize) - 1)\n        t0 = (arg0 << tempCount)\n        t1 = arg1 >> (OperandSize - tempCount)\n        res = Operators.ITEBV(OperandSize, tempCount == 0, arg0, t0 | t1)\n        res = res & MASK\n        dest.write(res)\n        if isinstance(tempCount, int) and tempCount == 0:\n            pass\n        else:\n            SIGN_MASK = 1 << (OperandSize - 1)\n            lastbit = 0 != ((arg0 << (tempCount - 1)) & SIGN_MASK)\n\n            cpu._set_shiftd_flags(OperandSize, arg0, res, lastbit, tempCount)", "response": "Double precision shift right."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _getMemoryBit(cpu, bitbase, bitoffset):\n        assert bitbase.type == 'memory'\n        assert bitbase.size >= bitoffset.size\n        addr = bitbase.address()\n        offt = Operators.SEXTEND(bitoffset.read(), bitoffset.size, bitbase.size)\n        offt_is_neg = offt >= (1 << (bitbase.size - 1))\n        offt_in_bytes = offt // 8\n        bitpos = offt % 8\n\n        new_addr = addr + Operators.ITEBV(bitbase.size, offt_is_neg, -offt_in_bytes, offt_in_bytes)\n        return (new_addr, bitpos)", "response": "Calculate the address and bit offset for a memory entry."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nbiting scan forward. Searches the source operand (second operand) for the least significant set bit (1 bit). If a least significant 1 bit is found, its bit index is stored in the destination operand (first operand). The source operand can be a register or a memory location; the destination operand is a register. The bit index is an unsigned offset from bit 0 of the source operand. If the contents source operand are 0, the contents of the destination operand is undefined:: IF SRC = 0 THEN ZF = 1; DEST is undefined; ELSE ZF = 0; temp = 0; WHILE Bit(SRC, temp) = 0 DO temp = temp + 1; DEST = temp; OD; FI; :param cpu: current CPU. :param dest: destination operand. :param src: source operand.", "response": "def BSF(cpu, dest, src):\n        \"\"\"\n        Bit scan forward.\n\n        Searches the source operand (second operand) for the least significant\n        set bit (1 bit). If a least significant 1 bit is found, its bit index\n        is stored in the destination operand (first operand). The source operand\n        can be a register or a memory location; the destination operand is a register.\n        The bit index is an unsigned offset from bit 0 of the source operand.\n        If the contents source operand are 0, the contents of the destination\n        operand is undefined::\n\n                    IF SRC  =  0\n                    THEN\n                        ZF  =  1;\n                        DEST is undefined;\n                    ELSE\n                        ZF  =  0;\n                        temp  =  0;\n                        WHILE Bit(SRC, temp)  =  0\n                        DO\n                            temp  =  temp + 1;\n                            DEST  =  temp;\n                        OD;\n                    FI;\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: source operand.\n        \"\"\"\n        value = src.read()\n        flag = Operators.EXTRACT(value, 0, 1) == 1\n        res = 0\n        for pos in range(1, src.size):\n            res = Operators.ITEBV(dest.size, flag, res, pos)\n            flag = Operators.OR(flag, Operators.EXTRACT(value, pos, 1) == 1)\n\n        cpu.ZF = value == 0\n        dest.write(Operators.ITEBV(dest.size, cpu.ZF, dest.read(), res))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef BSR(cpu, dest, src):\n        value = src.read()\n        flag = Operators.EXTRACT(value, src.size - 1, 1) == 1\n        res = 0\n\n        for pos in reversed(range(0, src.size)):\n            res = Operators.ITEBV(dest.size, flag, res, pos)\n            flag = Operators.OR(flag, (Operators.EXTRACT(value, pos, 1) == 1))\n\n        cpu.PF = cpu._calculate_parity_flag(res)\n        cpu.ZF = value == 0\n        dest.write(Operators.ITEBV(dest.size, cpu.ZF, dest.read(), res))", "response": "Bit scan for the most significant set bit in the source and destination memory locations."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nbiting test and complement.", "response": "def BTC(cpu, dest, src):\n        \"\"\"\n        Bit test and complement.\n\n        Selects the bit in a bit string (specified with the first operand, called\n        the bit base) at the bit-position designated by the bit offset operand\n        (second operand), stores the value of the bit in the CF flag, and complements\n        the selected bit in the bit string.\n\n        :param cpu: current CPU.\n        :param dest: bit base operand.\n        :param src: bit offset operand.\n        \"\"\"\n        if dest.type == 'register':\n            value = dest.read()\n            pos = src.read() % dest.size\n            cpu.CF = value & (1 << pos) == 1 << pos\n            dest.write(value ^ (1 << pos))\n        elif dest.type == 'memory':\n            addr, pos = cpu._getMemoryBit(dest, src)\n            base, size, ty = cpu.get_descriptor(cpu.DS)\n            addr += base\n            value = cpu.read_int(addr, 8)\n            cpu.CF = value & (1 << pos) == 1 << pos\n            value = value ^ (1 << pos)\n            cpu.write_int(addr, value, 8)\n        else:\n            raise NotImplementedError(f\"Unknown operand for BTC: {dest.type}\")"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CMPS(cpu, dest, src):\n        src_reg = {8: 'SI', 32: 'ESI', 64: 'RSI'}[cpu.address_bit_size]\n        dest_reg = {8: 'DI', 32: 'EDI', 64: 'RDI'}[cpu.address_bit_size]\n\n        base, _, ty = cpu.get_descriptor(cpu.DS)\n\n        src_addr = cpu.read_register(src_reg) + base\n        dest_addr = cpu.read_register(dest_reg) + base\n        size = dest.size\n\n        # Compare\n        arg1 = cpu.read_int(dest_addr, size)\n        arg0 = cpu.read_int(src_addr, size)\n        res = (arg0 - arg1) & ((1 << size) - 1)\n\n        cpu._calculate_CMP_flags(size, res, arg0, arg1)\n\n        #Advance EDI/ESI pointers\n        increment = Operators.ITEBV(cpu.address_bit_size, cpu.DF, -size // 8, size // 8)\n        cpu.write_register(src_reg, cpu.read_register(src_reg) + increment)\n        cpu.write_register(dest_reg, cpu.read_register(dest_reg) + increment)", "response": "Sets the status of the two sources in memory."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef LODS(cpu, dest, src):\n        src_reg = {8: 'SI', 32: 'ESI', 64: 'RSI'}[cpu.address_bit_size]\n        base, _, ty = cpu.get_descriptor(cpu.DS)\n\n        src_addr = cpu.read_register(src_reg) + base\n        size = dest.size\n\n        arg0 = cpu.read_int(src_addr, size)\n        dest.write(arg0)\n\n        increment = Operators.ITEBV(cpu.address_bit_size, cpu.DF, -size // 8, size // 8)\n        cpu.write_register(src_reg, cpu.read_register(src_reg) + increment)", "response": "Load a string from the source operand into the destination operand."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef MOVS(cpu, dest, src):\n        base, size, ty = cpu.get_descriptor(cpu.DS)\n        src_addr = src.address() + base\n        dest_addr = dest.address() + base\n\n        src_reg = src.mem.base\n        dest_reg = dest.mem.base\n        size = dest.size\n\n        # Copy the data\n        dest.write(src.read())\n\n        #Advance EDI/ESI pointers\n        increment = Operators.ITEBV(cpu.address_bit_size, cpu.DF, -size // 8, size // 8)\n        cpu.write_register(src_reg, cpu.read_register(src_reg) + increment)\n        cpu.write_register(dest_reg, cpu.read_register(dest_reg) + increment)", "response": "Moves data from string to string."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef SCAS(cpu, dest, src):\n        dest_reg = dest.reg\n        mem_reg = src.mem.base  # , src.type, src.read()\n        size = dest.size\n        arg0 = dest.read()\n        arg1 = src.read()\n        res = arg0 - arg1\n        cpu._calculate_CMP_flags(size, res, arg0, arg1)\n\n        increment = Operators.ITEBV(cpu.address_bit_size, cpu.DF, -size // 8, size // 8)\n        cpu.write_register(mem_reg, cpu.read_register(mem_reg) + increment)", "response": "Sets the status flags of the current object based on the contents of the source object."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nstores a byte word doubleword from the AL AX or EAX register of the source operand into the destination operand.", "response": "def STOS(cpu, dest, src):\n        \"\"\"\n        Stores String.\n\n        Stores a byte, word, or doubleword from the AL, AX, or EAX register,\n        respectively, into the destination operand. The destination operand is\n        a memory location, the address of which is read from either the ES:EDI\n        or the ES:DI registers (depending on the address-size attribute of the\n        instruction, 32 or 16, respectively). The ES segment cannot be overridden\n        with a segment override prefix.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: source operand.\n        \"\"\"\n        size = src.size\n        dest.write(src.read())\n        dest_reg = dest.mem.base\n        increment = Operators.ITEBV({'RDI': 64, 'EDI': 32, 'DI': 16}[dest_reg], cpu.DF, -size // 8, size // 8)\n        cpu.write_register(dest_reg, cpu.read_register(dest_reg) + increment)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ANDN(cpu, dest, src1, src2):\n        value = ~src1.read() & src2.read()\n        dest.write(value)\n        cpu.ZF = value == 0\n        cpu.SF = (value & (1 << dest.size)) != 0\n        cpu.OF = False\n        cpu.CF = False", "response": "Performs a bitwise AND of the two sources in the source table."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef SARX(cpu, dest, src, count):\n        OperandSize = dest.size\n        count = count.read()\n        countMask = {8: 0x1f,\n                     16: 0x1f,\n                     32: 0x1f,\n                     64: 0x3f}[OperandSize]\n        tempCount = count & countMask\n        tempDest = value = src.read()\n\n        sign = value & (1 << (OperandSize - 1))\n        while tempCount != 0:\n            cpu.CF = (value & 0x1) != 0  # LSB\n            value = (value >> 1) | sign\n            tempCount = tempCount - 1\n        res = dest.write(value)", "response": "The shift arithmetic right."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _PUNPCKL(cpu, dest, src, item_size):\n        assert dest.size == src.size\n        size = dest.size\n        dest_value = dest.read()\n        src_value = src.read()\n        mask = (1 << item_size) - 1\n        res = 0\n        count = 0\n        for pos in range(0, size // item_size):\n            if count >= size:\n                break\n            item0 = Operators.ZEXTEND((dest_value >> (pos * item_size)) & mask, size)\n            item1 = Operators.ZEXTEND((src_value >> (pos * item_size)) & mask, size)\n            res |= item0 << count\n            count += item_size\n            res |= item1 << count\n            count += item_size\n\n        dest.write(res)", "response": "Generic PUNPCKL Generic PUNPCKLGeneric."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef PSHUFW(cpu, op0, op1, op3):\n        size = op0.size\n        arg0 = op0.read()\n        arg1 = op1.read()\n        arg3 = Operators.ZEXTEND(op3.read(), size)\n        assert size == 64\n        arg0 |= ((arg1 >> ((arg3 >> 0) & 3 * 16)) & 0xffff)\n        arg0 |= ((arg1 >> ((arg3 >> 2) & 3 * 16)) & 0xffff) << 16\n        arg0 |= ((arg1 >> ((arg3 >> 4) & 3 * 16)) & 0xffff) << 32\n        arg0 |= ((arg1 >> ((arg3 >> 6) & 3 * 16)) & 0xffff) << 48\n        op0.write(arg0)", "response": "Packed shuffle words.\n\n        Copies doublewords from source operand (second operand) and inserts them in the destination operand\n        (first operand) at locations selected with the order operand (third operand).\n\n        :param cpu: current CPU.\n        :param op0: destination operand.\n        :param op1: source operand.\n        :param op3: order operand."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\npacking shuffle doublewords. Copies doublewords from source operand (second operand) and inserts them in the destination operand (first operand) at locations selected with the order operand (third operand). :param cpu: current CPU. :param op0: destination operand. :param op1: source operand. :param op3: order operand.", "response": "def PSHUFD(cpu, op0, op1, op3):\n        \"\"\"\n        Packed shuffle doublewords.\n\n        Copies doublewords from source operand (second operand) and inserts them in the destination operand\n        (first operand) at locations selected with the order operand (third operand).\n\n        :param cpu: current CPU.\n        :param op0: destination operand.\n        :param op1: source operand.\n        :param op3: order operand.\n         \"\"\"\n        size = op0.size\n        arg0 = op0.read()\n        arg1 = op1.read()\n        order = Operators.ZEXTEND(op3.read(), size)\n\n        arg0 = arg0 & 0xffffffffffffffffffffffffffffffff00000000000000000000000000000000\n        arg0 |= ((arg1 >> (((order >> 0) & 3) * 32)) & 0xffffffff)\n        arg0 |= ((arg1 >> (((order >> 2) & 3) * 32)) & 0xffffffff) << 32\n        arg0 |= ((arg1 >> (((order >> 4) & 3) * 32)) & 0xffffffff) << 64\n        arg0 |= ((arg1 >> (((order >> 6) & 3) * 32)) & 0xffffffff) << 96\n\n        op0.write(arg0)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef PMOVMSKB(cpu, op0, op1):\n        arg0 = op0.read()\n        arg1 = op1.read()\n\n        res = 0\n        for i in reversed(range(7, op1.size, 8)):\n            res = (res << 1) | ((arg1 >> i) & 1)\n        op0.write(Operators.EXTRACT(res, 0, op0.size))", "response": "Moves byte mask to general - purpose register."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef PSRLDQ(cpu, dest, src):\n        # TODO(yan): Verify the correctness of truncating SRC like this ( tests\n        # use '-1' as the value\n        temp = Operators.EXTRACT(src.read(), 0, 8)\n        temp = Operators.ITEBV(src.size, temp > 15, 16, temp)\n        dest.write(dest.read() >> (temp * 8))", "response": "Packs the destination and source memory registers right by the number of bytes specified in the count operand."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nmove with zero - extend.", "response": "def MOVZX(cpu, op0, op1):\n        \"\"\"\n        Moves with zero-extend.\n\n        Copies the contents of the source operand (register or memory location) to the destination\n        operand (register) and zero extends the value to 16 or 32 bits. The size of the converted value\n        depends on the operand-size attribute::\n\n                OP0  =  ZeroExtend(OP1);\n\n        :param cpu: current CPU.\n        :param op0: destination operand.\n        :param op1: source operand.\n        \"\"\"\n        op0.write(Operators.ZEXTEND(op1.read(), op0.size))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef MOVSX(cpu, op0, op1):\n        op0.write(Operators.SEXTEND(op1.read(), op1.size, op0.size))", "response": "Moving with sign - extension."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef CQO(cpu):\n        res = Operators.SEXTEND(cpu.RAX, 64, 128)\n        cpu.RAX = Operators.EXTRACT(res, 0, 64)\n        cpu.RDX = Operators.EXTRACT(res, 64, 64)", "response": "CQO - set the RAX and RDX bits of the current CPU."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting CPU. EAX to sign - extend of EAX", "response": "def CDQ(cpu):\n        \"\"\"\n        EDX:EAX = sign-extend of EAX\n        \"\"\"\n        cpu.EDX = Operators.EXTRACT(Operators.SEXTEND(cpu.EAX, 32, 64), 32, 32)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nconverting word to doubleword.", "response": "def CWDE(cpu):\n        \"\"\"\n        Converts word to doubleword.\n\n        ::\n            DX = sign-extend of AX.\n\n        :param cpu: current CPU.\n        \"\"\"\n        bit = Operators.EXTRACT(cpu.AX, 15, 1)\n        cpu.EAX = Operators.SEXTEND(cpu.AX, 16, 32)\n        cpu.EDX = Operators.SEXTEND(bit, 1, 32)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread the current value of the time - stamp counter into the RAX and RAX registers.", "response": "def RDTSC(cpu):\n        \"\"\"\n        Reads time-stamp counter.\n\n        Loads the current value of the processor's time-stamp counter into the\n        EDX:EAX registers.  The time-stamp counter is contained in a 64-bit\n        MSR. The high-order 32 bits of the MSR are loaded into the EDX\n        register, and the low-order 32 bits are loaded into the EAX register.\n        The processor increments the time-stamp counter MSR every clock cycle\n        and resets it to 0 whenever the processor is reset.\n\n        :param cpu: current CPU.\n        \"\"\"\n        val = cpu.icount\n        cpu.RAX = val & 0xffffffff\n        cpu.RDX = (val >> 32) & 0xffffffff"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef SYSCALL(cpu):\n        cpu.RCX = cpu.RIP\n        cpu.R11 = cpu.RFLAGS\n        raise Syscall()", "response": "This function is used to generate the call to interrupt or exception handler specified\n       ."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef MOVLPD(cpu, dest, src):\n        value = src.read()\n        if src.size == 64 and dest.size == 128:\n            value = (dest.read() & 0xffffffffffffffff0000000000000000) | Operators.ZEXTEND(value, 128)\n        dest.write(value)", "response": "This instruction moves a double - precision floating - point value from one CPU to another."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nmove high packed double-precision floating-point value. Moves a double-precision floating-point value from the source operand (second operand) and the destination operand (first operand). The source and destination operands can be an XMM register or a 64-bit memory location. This instruction allows double-precision floating-point values to be moved to and from the high quadword of an XMM register and memory. It cannot be used for register to register or memory to memory moves. When the destination operand is an XMM register, the low quadword of the register remains unchanged. :param cpu: current CPU. :param dest: destination operand. :param src: source operand.", "response": "def MOVHPD(cpu, dest, src):\n        \"\"\"\n        Moves high packed double-precision floating-point value.\n\n        Moves a double-precision floating-point value from the source operand (second operand) and the\n        destination operand (first operand). The source and destination operands can be an XMM register\n        or a 64-bit memory location. This instruction allows double-precision floating-point values to be moved\n        to and from the high quadword of an XMM register and memory. It cannot be used for register to\n        register or memory to memory moves. When the destination operand is an XMM register, the low quadword\n        of the register remains unchanged.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: source operand.\n        \"\"\"\n        if src.size == 128:\n            assert dest.size == 64\n            dest.write(Operators.EXTRACT(src.read(), 64, 64))\n        else:\n            assert src.size == 64 and dest.size == 128\n            value = Operators.EXTRACT(dest.read(), 0, 64)  # low part\n            dest.write(Operators.CONCAT(128, src.read(), value))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\npack subtract. Performs a SIMD subtract of the packed integers of the source operand (second operand) from the packed integers of the destination operand (first operand), and stores the packed integer results in the destination operand. The source operand can be an MMX(TM) technology register or a 64-bit memory location, or it can be an XMM register or a 128-bit memory location. The destination operand can be an MMX or an XMM register. The PSUBB instruction subtracts packed byte integers. When an individual result is too large or too small to be represented in a byte, the result is wrapped around and the low 8 bits are written to the destination element. :param cpu: current CPU. :param dest: destination operand. :param src: source operand.", "response": "def PSUBB(cpu, dest, src):\n        \"\"\"\n        Packed subtract.\n\n        Performs a SIMD subtract of the packed integers of the source operand (second operand) from the packed\n        integers of the destination operand (first operand), and stores the packed integer results in the\n        destination operand. The source operand can be an MMX(TM) technology register or a 64-bit memory location,\n        or it can be an XMM register or a 128-bit memory location. The destination operand can be an MMX or an XMM\n        register.\n        The PSUBB instruction subtracts packed byte integers. When an individual result is too large or too small\n        to be represented in a byte, the result is wrapped around and the low 8 bits are written to the\n        destination element.\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        :param src: source operand.\n        \"\"\"\n        result = []\n        value_a = dest.read()\n        value_b = src.read()\n        for i in reversed(range(0, dest.size, 8)):\n            a = Operators.EXTRACT(value_a, i, 8)\n            b = Operators.EXTRACT(value_b, i, 8)\n            result.append((a - b) & 0xff)\n        dest.write(Operators.CONCAT(8 * len(result), *result))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef POR(cpu, dest, src):\n        res = dest.write(dest.read() | src.read())", "response": "Performs a bitwise OR operation on the source and destination memory locations."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nperforms a bitwise OR operation on the source and destination memory locations.", "response": "def XORPS(cpu, dest, src):\n        \"\"\"\n        Performs a bitwise logical OR operation on the source operand (second operand) and the destination operand\n        (first operand) and stores the result in the destination operand. The source operand can be an MMX technology\n        register or a 64-bit memory location or it can be an XMM register or a 128-bit memory location. The destination\n        operand can be an MMX technology register or an XMM register. Each bit of the result is set to 1 if either\n        or both of the corresponding bits of the first and second operands are 1; otherwise, it is set to 0.\n        \"\"\"\n        res = dest.write(dest.read() ^ src.read())"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef VORPD(cpu, dest, src, src2):\n        res = dest.write(src.read() | src2.read())", "response": "A logical OR operation on the source and destination memory locations."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef VORPS(cpu, dest, src, src2):\n        res = dest.write(src.read() | src2.read())", "response": "Performs a bitwise OR operation on the source and second source operands and stores the result in the destination operand."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef MOVQ(cpu, dest, src):\n        # mmx to mmx or mmx to mem\n        if dest.size == src.size and dest.size == 64:\n            dest.write(src.read())\n        # two xmm regs\n        elif dest.size == src.size and dest.size == 128:\n            src_lo = Operators.EXTRACT(src.read(), 0, 64)\n            dest.write(Operators.ZEXTEND(src_lo, 128))\n        # mem to xmm\n        elif dest.size == 128 and src.size == 64:\n            dest.write(Operators.ZEXTEND(src.read(), dest.size))\n        # xmm to mem\n        elif dest.size == 64 and src.size == 128:\n            dest.write(Operators.EXTRACT(src.read(), 0, dest.size))\n        else:\n            msg = 'Invalid size in MOVQ'\n            logger.error(msg)\n            raise Exception(msg)", "response": "This function moves a quadword from one memory location to another."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef MOVSD(cpu, dest, src):\n        assert dest.type != 'memory' or src.type != 'memory'\n        value = Operators.EXTRACT(src.read(), 0, 64)\n        if dest.size > src.size:\n            value = Operators.ZEXTEND(value, dest.size)\n        dest.write(value)", "response": "Move Scalar Double - Precision Floating - Point Value"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef MOVSS(cpu, dest, src):\n        if dest.type == 'register' and src.type == 'register':\n            assert dest.size == 128 and src.size == 128\n            dest.write(dest.read() & ~0xffffffff | src.read() & 0xffffffff)\n        elif dest.type == 'memory':\n            assert src.type == 'register'\n            dest.write(Operators.EXTRACT(src.read(), 0, dest.size))\n        else:\n            assert src.type == 'memory' and dest.type == 'register'\n            assert src.size == 32 and dest.size == 128\n            dest.write(Operators.ZEXTEND(src.read(), 128))", "response": "This function moves a scalar single - precision floating - point value from one memory location to another."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nextract Packed Floating - Point Values from the source and write them to the destination.", "response": "def VEXTRACTF128(cpu, dest, src, offset):\n        \"\"\"Extract Packed Floating-Point Values\n\n        Extracts 128-bits of packed floating-point values from the source\n        operand (second operand) at an 128-bit offset from imm8[0] into the\n        destination operand (first operand). The destination may be either an\n        XMM register or an 128-bit memory location.\n        \"\"\"\n        offset = offset.read()\n        dest.write(Operators.EXTRACT(src.read(), offset * 128, (offset + 1) * 128))"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\npack Shift Left Logical Double Quadword", "response": "def PSLLDQ(cpu, dest, src):\n        \"\"\" Packed Shift Left Logical Double Quadword\n        Shifts the destination operand (first operand) to the left by the number\n         of bytes specified in the count operand (second operand). The empty low-order\n         bytes are cleared (set to all 0s). If the value specified by the count\n         operand is greater than 15, the destination operand is set to all 0s.\n         The destination operand is an XMM register. The count operand is an 8-bit\n         immediate.\n\n            TEMP  =  COUNT;\n            if (TEMP > 15) TEMP  =  16;\n            DEST  =  DEST << (TEMP * 8);\n        \"\"\"\n        count = Operators.ZEXTEND(src.read(), dest.size * 2)\n        byte_count = Operators.ITEBV(src.size * 2, count > 15, 16, count)\n        bit_count = byte_count * 8\n        val = Operators.ZEXTEND(dest.read(), dest.size * 2)\n        val = val << (Operators.ZEXTEND(bit_count, dest.size * 2))\n        dest.write(Operators.EXTRACT(val, 0, dest.size))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef PSRLQ(cpu, dest, src):\n\n        count = src.read()\n        count = Operators.ITEBV(src.size, Operators.UGT(count, 63), 64, count)\n        count = Operators.EXTRACT(count, 0, 64)\n        if dest.size == 64:\n            dest.write(dest.read() >> count)\n        else:\n            hi = Operators.EXTRACT(dest.read(), 64, 64) >> count\n            low = Operators.EXTRACT(dest.read(), 0, 64) >> count\n            dest.write(Operators.CONCAT(128, hi, low))", "response": "Shift Packed Data Right Logical\n            - > destination"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef VPSHUFB(cpu, op0, op1, op3):\n        size = op0.size\n        arg0 = op0.read()\n        arg1 = op1.read()\n        arg3 = Operators.ZEXTEND(op3.read(), size)\n\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 7, 1) == 1, 0, (arg1 >> ((arg3 >> 0) & 7 * 8)) & 0xff)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 15, 1) == 1, 0, ((arg1 >> ((arg3 >> 8) & 7 * 8)) & 0xff) << 8)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 23, 1) == 1, 0, ((arg1 >> ((arg3 >> 16) & 7 * 8)) & 0xff) << 16)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 31, 1) == 1, 0, ((arg1 >> ((arg3 >> 24) & 7 * 8)) & 0xff) << 24)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 39, 1) == 1, 0, ((arg1 >> ((arg3 >> 32) & 7 * 8)) & 0xff) << 32)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 47, 1) == 1, 0, ((arg1 >> ((arg3 >> 40) & 7 * 8)) & 0xff) << 40)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 55, 1) == 1, 0, ((arg1 >> ((arg3 >> 48) & 7 * 8)) & 0xff) << 48)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 63, 1) == 1, 0, ((arg1 >> ((arg3 >> 56) & 7 * 8)) & 0xff) << 56)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 71, 1) == 1, 0, ((arg1 >> ((arg3 >> 64) & 7 * 8)) & 0xff) << 64)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 79, 1) == 1, 0, ((arg1 >> ((arg3 >> 72) & 7 * 8)) & 0xff) << 72)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 87, 1) == 1, 0, ((arg1 >> ((arg3 >> 80) & 7 * 8)) & 0xff) << 80)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 95, 1) == 1, 0, ((arg1 >> ((arg3 >> 88) & 7 * 8)) & 0xff) << 88)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 103, 1) == 1, 0, ((arg1 >> ((arg3 >> 96) & 7 * 8)) & 0xff) << 96)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 111, 1) == 1, 0, ((arg1 >> ((arg3 >> 104) & 7 * 8)) & 0xff) << 104)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 119, 1) == 1, 0, ((arg1 >> ((arg3 >> 112) & 7 * 8)) & 0xff) << 112)\n        arg0 |= Operators.ITEBV(size, Operators.EXTRACT(arg3, 127, 1) == 1, 0, ((arg1 >> ((arg3 >> 120) & 7 * 8)) & 0xff) << 120)\n        op0.write(arg0)", "response": "Packed shuffle bytes.\n\n        Copies bytes from source operand (second operand) and inserts them in the destination operand\n        (first operand) at locations selected with the order operand (third operand).\n\n        :param cpu: current CPU.\n        :param op0: destination operand.\n        :param op1: source operand.\n        :param op3: order operand."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef XLATB(cpu):\n        cpu.AL = cpu.read_int(cpu.RBX + Operators.ZEXTEND(cpu.AL, 64), 8)", "response": "A table look - up translation."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ntabling look-up translation. Locates a byte entry in a table in memory, using the contents of the AL register as a table index, then copies the contents of the table entry back into the AL register. The index in the AL register is treated as an unsigned integer. The XLAT and XLATB instructions get the base address of the table in memory from either the DS:EBX or the DS:BX registers. In 64-bit mode, operation is similar to that in legacy or compatibility mode. AL is used to specify the table index (the operand size is fixed at 8 bits). RBX, however, is used to specify the table's base address:: IF address_bit_size = 16 THEN AL = (DS:BX + ZeroExtend(AL)); ELSE IF (address_bit_size = 32) AL = (DS:EBX + ZeroExtend(AL)); FI; ELSE (address_bit_size = 64) AL = (RBX + ZeroExtend(AL)); FI; :param cpu: current CPU. :param dest: destination operand.", "response": "def XLATB(cpu):\n        \"\"\"\n        Table look-up translation.\n\n        Locates a byte entry in a table in memory, using the contents of the\n        AL register as a table index, then copies the contents of the table entry\n        back into the AL register. The index in the AL register is treated as\n        an unsigned integer. The XLAT and XLATB instructions get the base address\n        of the table in memory from either the DS:EBX or the DS:BX registers.\n        In 64-bit mode, operation is similar to that in legacy or compatibility mode.\n        AL is used to specify the table index (the operand size is fixed at 8 bits).\n        RBX, however, is used to specify the table's base address::\n\n                IF address_bit_size = 16\n                THEN\n                    AL = (DS:BX + ZeroExtend(AL));\n                ELSE IF (address_bit_size = 32)\n                    AL = (DS:EBX + ZeroExtend(AL)); FI;\n                ELSE (address_bit_size = 64)\n                    AL = (RBX + ZeroExtend(AL));\n                FI;\n\n        :param cpu: current CPU.\n        :param dest: destination operand.\n        \"\"\"\n        cpu.AL = cpu.read_int(cpu.EBX + Operators.ZEXTEND(cpu.AL, 32), 8)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef constrain(self, constraint):\n        constraint = self.migrate_expression(constraint)\n        self._constraints.add(constraint)", "response": "Add a constraint to the set of constraints."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef new_symbolic_buffer(self, nbytes, **options):\n        label = options.get('label')\n        avoid_collisions = False\n        if label is None:\n            label = 'buffer'\n            avoid_collisions = True\n        taint = options.get('taint', frozenset())\n        expr = self._constraints.new_array(name=label, index_max=nbytes, value_bits=8, taint=taint, avoid_collisions=avoid_collisions)\n        self._input_symbols.append(expr)\n\n        if options.get('cstring', False):\n            for i in range(nbytes - 1):\n                self._constraints.add(expr[i] != 0)\n\n        return expr", "response": "Create and return a symbolic buffer of length nbytes."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef new_symbolic_value(self, nbits, label=None, taint=frozenset()):\n        assert nbits in (1, 4, 8, 16, 32, 64, 128, 256)\n        avoid_collisions = False\n        if label is None:\n            label = 'val'\n            avoid_collisions = True\n\n        expr = self._constraints.new_bitvec(nbits, name=label, taint=taint, avoid_collisions=avoid_collisions)\n        self._input_symbols.append(expr)\n        return expr", "response": "Create and return a symbolic value that is nbits bits wide."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef concretize(self, symbolic, policy, maxcount=7):\n        assert self.constraints == self.platform.constraints\n        symbolic = self.migrate_expression(symbolic)\n\n        vals = []\n        if policy == 'MINMAX':\n            vals = self._solver.minmax(self._constraints, symbolic)\n        elif policy == 'MAX':\n            vals = self._solver.max(self._constraints, symbolic)\n        elif policy == 'MIN':\n            vals = self._solver.min(self._constraints, symbolic)\n        elif policy == 'SAMPLED':\n            m, M = self._solver.minmax(self._constraints, symbolic)\n            vals += [m, M]\n            if M - m > 3:\n                if self._solver.can_be_true(self._constraints, symbolic == (m + M) // 2):\n                    vals.append((m + M) // 2)\n            if M - m > 100:\n                for i in (0, 1, 2, 5, 32, 64, 128, 320):\n                    if self._solver.can_be_true(self._constraints, symbolic == m + i):\n                        vals.append(m + i)\n                    if maxcount <= len(vals):\n                        break\n            if M - m > 1000 and maxcount > len(vals):\n                vals += self._solver.get_all_values(self._constraints, symbolic,\n                                                    maxcnt=maxcount - len(vals), silent=True)\n        elif policy == 'ONE':\n            vals = [self._solver.get_value(self._constraints, symbolic)]\n        else:\n            assert policy == 'ALL'\n            vals = solver.get_all_values(self._constraints, symbolic, maxcnt=maxcount, silent=True)\n\n        return tuple(set(vals))", "response": "This method concretizes a set of solutions for a given symbolic."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsolves a symbolic value into one solution.", "response": "def solve_one(self, expr, constrain=False):\n        \"\"\"\n        Concretize a symbolic :class:`~manticore.core.smtlib.expression.Expression` into\n        one solution.\n\n        :param manticore.core.smtlib.Expression expr: Symbolic value to concretize\n        :param bool constrain: If True, constrain expr to concretized value\n        :return: Concrete value\n        :rtype: int\n        \"\"\"\n        expr = self.migrate_expression(expr)\n        value = self._solver.get_value(self._constraints, expr)\n        if constrain:\n            self.constrain(expr == value)\n        #Include forgiveness here\n        if isinstance(value, bytearray):\n            value = bytes(value)\n        return value"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsolving a symbolic value into a list of n possible values.", "response": "def solve_n(self, expr, nsolves):\n        \"\"\"\n        Concretize a symbolic :class:`~manticore.core.smtlib.expression.Expression` into\n        `nsolves` solutions.\n\n        :param manticore.core.smtlib.Expression expr: Symbolic value to concretize\n        :return: Concrete value\n        :rtype: list[int]\n        \"\"\"\n        expr = self.migrate_expression(expr)\n        return self._solver.get_all_values(self._constraints, expr, nsolves, silent=True)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef solve_max(self, expr):\n        if isinstance(expr, int):\n            return expr\n        expr = self.migrate_expression(expr)\n        return self._solver.max(self._constraints, expr)", "response": "Solves a symbolic value into\n        its maximum solution."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsolve a symbolic value into its minimum solution.", "response": "def solve_min(self, expr):\n        \"\"\"\n        Solves a symbolic :class:`~manticore.core.smtlib.expression.Expression` into\n        its minimum solution\n\n        :param manticore.core.smtlib.Expression expr: Symbolic value to solve\n        :return: Concrete value\n        :rtype: list[int]\n        \"\"\"\n        if isinstance(expr, int):\n            return expr\n        expr = self.migrate_expression(expr)\n        return self._solver.min(self._constraints, expr)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef solve_minmax(self, expr):\n        if isinstance(expr, int):\n            return expr\n        expr = self.migrate_expression(expr)\n        return self._solver.minmax(self._constraints, expr)", "response": "Solves a symbolic value into\n        its minimum and maximun solution. Only defined for bitvects."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef solve_buffer(self, addr, nbytes, constrain=False):\n        buffer = self.cpu.read_bytes(addr, nbytes)\n        result = []\n        with self._constraints as temp_cs:\n            cs_to_use = self.constraints if constrain else temp_cs\n            for c in buffer:\n                result.append(self._solver.get_value(cs_to_use, c))\n                cs_to_use.add(c == result[-1])\n        return result", "response": "Reads nbytes of symbolic data from memory at addr and concretizes it."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nmark parts of a buffer as symbolic (demarked by the wildcard byte) :param str data: The string to symbolicate. If no wildcard bytes are provided, this is the identity function on the first argument. :param str label: The label to assign to the value :param str wildcard: The byte that is considered a wildcard :param bool string: Ensure bytes returned can not be NULL :param taint: Taint identifier of the symbolicated data :type taint: tuple or frozenset :return: If data does not contain any wildcard bytes, data itself. Otherwise, a list of values derived from data. Non-wildcard bytes are kept as is, wildcard bytes are replaced by Expression objects.", "response": "def symbolicate_buffer(self, data, label='INPUT', wildcard='+', string=False, taint=frozenset()):\n        \"\"\"Mark parts of a buffer as symbolic (demarked by the wildcard byte)\n\n        :param str data: The string to symbolicate. If no wildcard bytes are provided,\n                this is the identity function on the first argument.\n        :param str label: The label to assign to the value\n        :param str wildcard: The byte that is considered a wildcard\n        :param bool string: Ensure bytes returned can not be NULL\n        :param taint: Taint identifier of the symbolicated data\n        :type taint: tuple or frozenset\n\n        :return: If data does not contain any wildcard bytes, data itself. Otherwise,\n            a list of values derived from data. Non-wildcard bytes are kept as\n            is, wildcard bytes are replaced by Expression objects.\n        \"\"\"\n        if wildcard in data:\n            size = len(data)\n            symb = self._constraints.new_array(name=label, index_max=size, taint=taint, avoid_collisions=True)\n            self._input_symbols.append(symb)\n\n            tmp = []\n            for i in range(size):\n                if data[i] == wildcard:\n                    tmp.append(symb[i])\n                else:\n                    tmp.append(data[i])\n\n            data = tmp\n\n        if string:\n            for b in data:\n                if issymbolic(b):\n                    self._constraints.add(b != 0)\n                else:\n                    assert b != 0\n        return data"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _create_emulated_mapping(self, uc, address):\n\n        m = self._cpu.memory.map_containing(address)\n\n        permissions = UC_PROT_NONE\n        if 'r' in m.perms:\n            permissions |= UC_PROT_READ\n        if 'w' in m.perms:\n            permissions |= UC_PROT_WRITE\n        if 'x' in m.perms:\n            permissions |= UC_PROT_EXEC\n\n        uc.mem_map(m.start, len(m), permissions)\n\n        self._should_be_mapped[m.start] = (len(m), permissions)\n\n        return m", "response": "Create a mapping in Unicorn and note that we ll need it if we retry."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _hook_xfer_mem(self, uc, access, address, size, value, data):\n        assert access in (UC_MEM_WRITE, UC_MEM_READ, UC_MEM_FETCH)\n\n        if access == UC_MEM_WRITE:\n            self._cpu.write_int(address, value, size * 8)\n\n        # If client code is attempting to read a value, we need to bring it\n        # in from Manticore state. If we try to mem_write it here, Unicorn\n        # will segfault. We add the value to a list of things that need to\n        # be written, and ask to restart the emulation.\n        elif access == UC_MEM_READ:\n            value = self._cpu.read_bytes(address, size)\n\n            if address in self._should_be_written:\n                return True\n\n            self._should_be_written[address] = value\n\n            self._should_try_again = True\n            return False\n\n        return True", "response": "Handle memory operations from unicorn."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nhooks for unmapped region mappings.", "response": "def _hook_unmapped(self, uc, access, address, size, value, data):\n        \"\"\"\n        We hit an unmapped region; map it into unicorn.\n        \"\"\"\n\n        try:\n            m = self._create_emulated_mapping(uc, address)\n        except MemoryException as e:\n            self._to_raise = e\n            self._should_try_again = False\n            return False\n\n        self._should_try_again = True\n        return False"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _interrupt(self, uc, number, data):\n\n        from ..native.cpu.abstractcpu import Interruption  # prevent circular imports\n        self._to_raise = Interruption(number)\n        return True", "response": "Handle software interrupt (SVC/INT)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nemulates a single instruction.", "response": "def emulate(self, instruction):\n        \"\"\"\n        Emulate a single instruction.\n        \"\"\"\n\n        # The emulation might restart if Unicorn needs to bring in a memory map\n        # or bring a value from Manticore state.\n        while True:\n\n            self.reset()\n\n            # Establish Manticore state, potentially from past emulation\n            # attempts\n            for base in self._should_be_mapped:\n                size, perms = self._should_be_mapped[base]\n                self._emu.mem_map(base, size, perms)\n\n            for address, values in self._should_be_written.items():\n                for offset, byte in enumerate(values, start=address):\n                    if issymbolic(byte):\n                        from ..native.cpu.abstractcpu import ConcretizeMemory\n                        raise ConcretizeMemory(self._cpu.memory, offset, 8,\n                                               \"Concretizing for emulation\")\n\n                self._emu.mem_write(address, b''.join(values))\n\n            # Try emulation\n            self._should_try_again = False\n\n            self._step(instruction)\n\n            if not self._should_try_again:\n                break"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _step(self, instruction):\n        logger.debug(\"0x%x:\\t%s\\t%s\"\n                     % (instruction.address, instruction.mnemonic, instruction.op_str))\n\n        registers = set(self._cpu.canonical_registers)\n\n        # Refer to EFLAGS instead of individual flags for x86\n        if self._cpu.arch == CS_ARCH_X86:\n            # The last 8 canonical registers of x86 are individual flags; replace\n            # with the eflags\n            registers -= set(['CF', 'PF', 'AF', 'ZF', 'SF', 'IF', 'DF', 'OF'])\n            registers.add('EFLAGS')\n\n            # TODO(mark): Unicorn 1.0.1 does not support reading YMM registers,\n            # and simply returns back zero. If a unicorn emulated instruction writes to an\n            # XMM reg, we will read back the corresponding YMM register, resulting in an\n            # incorrect zero value being actually written to the XMM register. This is\n            # fixed in Unicorn PR #819, so when that is included in a release, delete\n            # these two lines.\n            registers -= set(['YMM0', 'YMM1', 'YMM2', 'YMM3', 'YMM4', 'YMM5', 'YMM6', 'YMM7',\n                              'YMM8', 'YMM9', 'YMM10', 'YMM11', 'YMM12', 'YMM13', 'YMM14', 'YMM15'])\n            registers |= set(['XMM0', 'XMM1', 'XMM2', 'XMM3', 'XMM4', 'XMM5', 'XMM6', 'XMM7',\n                              'XMM8', 'XMM9', 'XMM10', 'XMM11', 'XMM12', 'XMM13', 'XMM14', 'XMM15'])\n\n        # XXX(yan): This concretizes the entire register state. This is overly\n        # aggressive. Once capstone adds consistent support for accessing\n        # referred registers, make this only concretize those registers being\n        # read from.\n        for reg in registers:\n            val = self._cpu.read_register(reg)\n            if issymbolic(val):\n                from ..native.cpu.abstractcpu import ConcretizeRegister\n                raise ConcretizeRegister(self._cpu, reg, \"Concretizing for emulation.\",\n                                         policy='ONE')\n            self._emu.reg_write(self._to_unicorn_id(reg), val)\n\n        # Bring in the instruction itself\n        instruction = self._cpu.decode_instruction(self._cpu.PC)\n        text_bytes = self._cpu.read_bytes(self._cpu.PC, instruction.size)\n        self._emu.mem_write(self._cpu.PC, b''.join(text_bytes))\n\n        self._emu.hook_add(UC_HOOK_MEM_READ_UNMAPPED, self._hook_unmapped)\n        self._emu.hook_add(UC_HOOK_MEM_WRITE_UNMAPPED, self._hook_unmapped)\n        self._emu.hook_add(UC_HOOK_MEM_FETCH_UNMAPPED, self._hook_unmapped)\n        self._emu.hook_add(UC_HOOK_MEM_READ, self._hook_xfer_mem)\n        self._emu.hook_add(UC_HOOK_MEM_WRITE, self._hook_xfer_mem)\n        self._emu.hook_add(UC_HOOK_INTR, self._interrupt)\n\n        saved_PC = self._cpu.PC\n\n        try:\n            pc = self._cpu.PC\n            if self._cpu.arch == CS_ARCH_ARM and self._uc_mode == UC_MODE_THUMB:\n                pc |= 1\n            self._emu.emu_start(pc, self._cpu.PC + instruction.size, count=1)\n        except UcError as e:\n            # We request re-execution by signaling error; if we we didn't set\n            # _should_try_again, it was likely an actual error\n            if not self._should_try_again:\n                raise\n\n        if self._should_try_again:\n            return\n\n        if logger.isEnabledFor(logging.DEBUG):\n            logger.debug(\"=\" * 10)\n            for register in self._cpu.canonical_registers:\n                logger.debug(\n                    f\"Register {register:3s}  \"\n                    f\"Manticore: {self._cpu.read_register(register):08x}, \"\n                    f\"Unicorn {self._emu.reg_read(self._to_unicorn_id(register)):08x}\"\n                )\n            logger.debug(\">\" * 10)\n\n        # Bring back Unicorn registers to Manticore\n        for reg in registers:\n            val = self._emu.reg_read(self._to_unicorn_id(reg))\n            self._cpu.write_register(reg, val)\n\n        # Unicorn hack. On single step, unicorn wont advance the PC register\n        mu_pc = self.get_unicorn_pc()\n        if saved_PC == mu_pc:\n            self._cpu.PC = saved_PC + instruction.size\n\n        # Raise the exception from a hook that Unicorn would have eaten\n        if self._to_raise:\n            raise self._to_raise\n\n        return", "response": "A single instruction at executing an unicorn internal state machine."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef rlp_encode(item):\n    if item is None or item == 0:\n        ret = b'\\x80'\n    elif isinstance(item, str):\n        ret = rlp_encode(item.encode('utf8'))\n    elif isinstance(item, (bytearray, bytes)):\n        if len(item) == 1 and item[0] < 0x80:\n            # For a single byte whose value is in the [0x00, 0x7f] range, that byte is its own RLP encoding.\n            ret = item\n        else:\n            ret = encode_length(len(item), 0x80) + item\n    elif isinstance(item, collections.abc.Sequence):\n        output = b''.join(map(rlp_encode, item))\n        ret = encode_length(len(output), 0xC0) + output\n    elif isinstance(item, int):\n        ret = rlp_encode(int_to_bytes(item))\n    else:\n        raise Exception(\"Cannot encode object of type %s\" % type(item).__name__)\n    return ret", "response": "r Encode the object to be encoded in RLP."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef must_be_true(self, constraints, expression) -> bool:\n        solutions = self.get_all_values(constraints, expression, maxcnt=2, silent=True)\n        return solutions == [True]", "response": "Check if expression is True and that it can not be False with current constraints"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef max(self, constraints, X: BitVec, M=10000):\n        assert isinstance(X, BitVec)\n        return self.optimize(constraints, X, 'maximize', M)", "response": "Iteratively finds the maximum value for a symbol within given constraints."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef minmax(self, constraints, x, iters=10000):\n        if issymbolic(x):\n            m = self.min(constraints, x, iters)\n            M = self.max(constraints, x, iters)\n            return m, M\n        else:\n            return x, x", "response": "Returns the min and max possible values for x within given constraints"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _solver_version(self) -> Version:\n        self._reset()\n        if self._received_version is None:\n            self._send('(get-info :version)')\n            self._received_version = self._recv()\n        key, version = shlex.split(self._received_version[1:-1])\n        return Version(*map(int, version.split('.')))", "response": "Parse the version from z3"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nspawning z3 solver process", "response": "def _start_proc(self):\n        \"\"\"Spawns z3 solver process\"\"\"\n        assert '_proc' not in dir(self) or self._proc is None\n        try:\n            self._proc = Popen(shlex.split(self._command), stdin=PIPE, stdout=PIPE, bufsize=0, universal_newlines=True)\n        except OSError as e:\n            print(e, \"Probably too many cached expressions? visitors._cache...\")\n            # Z3 was removed from the system in the middle of operation\n            raise Z3NotFoundError  # TODO(mark) don't catch this exception in two places\n\n        # run solver specific initializations\n        for cfg in self._init:\n            self._send(cfg)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _stop_proc(self):\n        if self._proc is None:\n            return\n        if self._proc.returncode is None:\n            try:\n                self._send(\"(exit)\")\n            except (SolverError, IOError) as e:\n                # z3 was too fast to close\n                logger.debug(str(e))\n            finally:\n                try:\n                    self._proc.stdin.close()\n                except IOError as e:\n                    logger.debug(str(e))\n                try:\n                    self._proc.stdout.close()\n                except IOError as e:\n                    logger.debug(str(e))\n                self._proc.kill()\n                # Wait for termination, to avoid zombies.\n                self._proc.wait()\n\n        self._proc: Popen = None", "response": "Stops the z3 solver process by sending an exit command to it and then waiting till the process terminates."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsend a string to the solver.", "response": "def _send(self, cmd: str):\n        \"\"\"\n        Send a string to the solver.\n\n        :param cmd: a SMTLIBv2 command (ex. (check-sat))\n        \"\"\"\n        logger.debug('>%s', cmd)\n        try:\n            self._proc.stdout.flush()\n            self._proc.stdin.write(f'{cmd}\\n')\n        except IOError as e:\n            raise SolverError(str(e))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _recv(self) -> str:\n        buf, left, right = self.__readline_and_count()\n        bufl = [buf]\n\n        while left != right:\n            buf, l, r = self.__readline_and_count()\n            bufl.append(buf)\n            left += l\n            right += r\n\n        buf = ''.join(bufl).strip()\n\n        logger.debug('<%s', buf)\n        if '(error' in bufl[0]:\n            raise Exception(f\"Error in smtlib: {bufl[0]}\")\n        return buf", "response": "Reads the response from the solver"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _is_sat(self) -> bool:\n        logger.debug(\"Solver.check() \")\n        start = time.time()\n        self._send('(check-sat)')\n        status = self._recv()\n        logger.debug(\"Check took %s seconds (%s)\", time.time() - start, status)\n        if status not in ('sat', 'unsat', 'unknown'):\n            raise SolverError(status)\n        if consider_unknown_as_unsat:\n            if status == 'unknown':\n                logger.info('Found an unknown core, probably a solver timeout')\n                status = 'unsat'\n\n        if status == 'unknown':\n            raise SolverUnknown(status)\n\n        return status == 'sat'", "response": "Check the satisfiability of the current state of the current system."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _assert(self, expression: Bool):\n        assert isinstance(expression, Bool)\n        smtlib = translate_to_smtlib(expression)\n        self._send('(assert %s)' % smtlib)", "response": "Auxiliary method to send an assert"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _getvalue(self, expression):\n        if not issymbolic(expression):\n            return expression\n        assert isinstance(expression, Variable)\n\n        if isinstance(expression, Array):\n            result = bytearray()\n            for c in expression:\n                expression_str = translate_to_smtlib(c)\n                self._send('(get-value (%s))' % expression_str)\n                response = self._recv()\n                result.append(int('0x{:s}'.format(response.split(expression_str)[1][3:-2]), 16))\n            return bytes(result)\n        else:\n            self._send('(get-value (%s))' % expression.name)\n            ret = self._recv()\n            assert ret.startswith('((') and ret.endswith('))'), ret\n\n            if isinstance(expression, Bool):\n                return {'true': True, 'false': False}[ret[2:-2].split(' ')[1]]\n            elif isinstance(expression, BitVec):\n                pattern, base = self._get_value_fmt\n                m = pattern.match(ret)\n                expr, value = m.group('expr'), m.group('value')\n                return int(value, base)\n\n        raise NotImplementedError(\"_getvalue only implemented for Bool and BitVec\")", "response": "Ask the solver for one possible assignment for given expression."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef can_be_true(self, constraints, expression):\n        if isinstance(expression, bool):\n            if not expression:\n                return expression\n            else:\n                # if True check if constraints are feasible\n                self._reset(constraints)\n                return self._is_sat()\n        assert isinstance(expression, Bool)\n\n        with constraints as temp_cs:\n            temp_cs.add(expression)\n            self._reset(temp_cs.to_string(related_to=expression))\n            return self._is_sat()", "response": "Check if two potentially symbolic values can be equal"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list with all possible values for the symbol x", "response": "def get_all_values(self, constraints, expression, maxcnt=None, silent=False):\n        \"\"\"Returns a list with all the possible values for the symbol x\"\"\"\n        if not isinstance(expression, Expression):\n            return [expression]\n        assert isinstance(constraints, ConstraintSet)\n        assert isinstance(expression, Expression)\n        expression = simplify(expression)\n        if maxcnt is None:\n            maxcnt = consts.maxsolutions\n\n        with constraints as temp_cs:\n            if isinstance(expression, Bool):\n                var = temp_cs.new_bool()\n            elif isinstance(expression, BitVec):\n                var = temp_cs.new_bitvec(expression.size)\n            elif isinstance(expression, Array):\n                var = temp_cs.new_array(index_max=expression.index_max, value_bits=expression.value_bits, taint=expression.taint).array\n            else:\n                raise NotImplementedError(f\"get_all_values only implemented for {type(expression)} expression type.\")\n\n            temp_cs.add(var == expression)\n            self._reset(temp_cs.to_string(related_to=var))\n\n            result = []\n\n            while self._is_sat():\n                value = self._getvalue(var)\n                result.append(value)\n                self._assert(var != value)\n\n                if len(result) >= maxcnt:\n                    if silent:\n                        # do not throw an exception if set to silent\n                        # Default is not silent, assume user knows\n                        # what they are doing and will check the size\n                        # of returned vals list (previous smtlib behavior)\n                        break\n                    else:\n                        raise TooManySolutions(result)\n\n            return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nasks the solver for one possible result of given expression.", "response": "def get_value(self, constraints, expression):\n        \"\"\"\n        Ask the solver for one possible result of given expression using given set of constraints.\n        \"\"\"\n        if not issymbolic(expression):\n            return expression\n        assert isinstance(expression, (Bool, BitVec, Array))\n        with constraints as temp_cs:\n            if isinstance(expression, Bool):\n                var = temp_cs.new_bool()\n            elif isinstance(expression, BitVec):\n                var = temp_cs.new_bitvec(expression.size)\n            elif isinstance(expression, Array):\n                var = []\n                result = []\n                for i in range(expression.index_max):\n                    subvar = temp_cs.new_bitvec(expression.value_bits)\n                    var.append(subvar)\n                    temp_cs.add(subvar == simplify(expression[i]))\n\n                self._reset(temp_cs)\n                if not self._is_sat():\n                    raise SolverError('Model is not available')\n\n                for i in range(expression.index_max):\n                    self._send('(get-value (%s))' % var[i].name)\n                    ret = self._recv()\n                    assert ret.startswith('((') and ret.endswith('))')\n                    pattern, base = self._get_value_fmt\n                    m = pattern.match(ret)\n                    expr, value = m.group('expr'), m.group('value')\n                    result.append(int(value, base))\n                return bytes(result)\n\n            temp_cs.add(var == expression)\n\n            self._reset(temp_cs)\n\n        if not self._is_sat():\n            raise SolverError('Model is not available')\n\n        self._send('(get-value (%s))' % var.name)\n        ret = self._recv()\n        if not (ret.startswith('((') and ret.endswith('))')):\n            raise SolverError('SMTLIB error parsing response: %s' % ret)\n\n        if isinstance(expression, Bool):\n            return {'true': True, 'false': False}[ret[2:-2].split(' ')[1]]\n        if isinstance(expression, BitVec):\n            pattern, base = self._get_value_fmt\n            m = pattern.match(ret)\n            expr, value = m.group('expr'), m.group('value')\n            return int(value, base)\n        raise NotImplementedError(\"get_value only implemented for Bool and BitVec\")"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef summarized_name(self, name):\n        components = name.split('.')\n        prefix = '.'.join(c[0] for c in components[:-1])\n        return f'{prefix}.{components[-1]}'", "response": "Produce a summarized record name by splitting it into a prefix and suffix."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef colored_level_name(self, levelname):\n        if self.colors_disabled:\n            return self.plain_levelname_format.format(levelname)\n        else:\n            return self.colored_levelname_format.format(self.color_map[levelname], levelname)", "response": "Colors the logging level in the log record"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef strlen(state, s):\n\n    cpu = state.cpu\n\n    if issymbolic(s):\n        raise ConcretizeArgument(state.cpu, 1)\n\n    zero_idx = _find_zero(cpu, state.constraints, s)\n\n    ret = zero_idx\n\n    for offset in range(zero_idx - 1, -1, -1):\n        byt = cpu.read_int(s + offset, 8)\n        if issymbolic(byt):\n            ret = ITEBV(cpu.address_bit_size, byt == 0, offset, ret)\n\n    return ret", "response": "Walks from the end of string and returns the length of the string."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef all_events(cls):\n        all_evts = set()\n        for cls, evts in cls.__all_events__.items():\n            all_evts.update(evts)\n        return all_evts", "response": "Return all events that all subclasses have so far registered to publish."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the shared context.", "response": "def context(self):\n        \"\"\" Convenient access to shared context \"\"\"\n        if self._context is not None:\n            return self._context\n        else:\n            logger.warning(\"Using shared context without a lock\")\n            return self._executor._shared_context"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a pstats. Stats instance with profiling results if run was called with should_profile = True. Otherwise returns None.", "response": "def get_profiling_stats(self):\n        \"\"\"\n        Returns a pstat.Stats instance with profiling results if `run` was called with `should_profile=True`.\n        Otherwise, returns `None`.\n        \"\"\"\n        profile_file_path = os.path.join(self.workspace, 'profiling.bin')\n        try:\n            return pstats.Stats(profile_file_path)\n        except Exception as e:\n            logger.debug(f'Failed to get profiling stats: {e}')\n            return None"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef run(self, procs=1, timeout=None, should_profile=False):\n        assert not self.running, \"Manticore is already running.\"\n        self._start_run()\n\n        self._last_run_stats['time_started'] = time.time()\n        with self.shutdown_timeout(timeout):\n            self._start_workers(procs, profiling=should_profile)\n\n            self._join_workers()\n        self._finish_run(profiling=should_profile)", "response": "Runs analysis.\n\n        :param int procs: Number of parallel worker processes\n        :param timeout: Analysis timeout, in seconds"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndispatches execution into one of Manticore s engines evm or native.", "response": "def main():\n    \"\"\"\n    Dispatches execution into one of Manticore's engines: evm or native.\n    \"\"\"\n    args = parse_arguments()\n\n    if args.no_colors:\n        log.disable_colors()\n\n    sys.setrecursionlimit(consts.recursionlimit)\n\n    ManticoreBase.verbosity(args.v)\n\n    if args.argv[0].endswith('.sol'):\n        ethereum_main(args, logger)\n    else:\n        install_helper.ensure_native_deps()\n        native_main(args, logger)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef GetNBits(value, nbits):\n    # NOP if sizes are the same\n    if isinstance(value, int):\n        return Operators.EXTRACT(value, 0, nbits)\n    elif isinstance(value, BitVec):\n        if value.size < nbits:\n            return Operators.ZEXTEND(value, nbits)\n        else:\n            return Operators.EXTRACT(value, 0, nbits)", "response": "Get the first nbits bits from value."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef SInt(value, width):\n    return Operators.ITEBV(width, Bit(value, width - 1) == 1,\n                           GetNBits(value, width) - 2**width,\n                           GetNBits(value, width))", "response": "Convert a bitstring value of width bits to a signed integer\n    representation."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef LSL_C(value, amount, width):\n    assert amount > 0\n    value = Operators.ZEXTEND(value, width * 2)\n    shifted = value << amount\n    result = GetNBits(shifted, width)\n    carry = Bit(shifted, width)\n    return (result, carry)", "response": "This function performs the ARM LSL_C operation."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef LSL(value, amount, width):\n    if amount == 0:\n        return value\n\n    result, _ = LSL_C(value, amount, width)\n    return result", "response": "This function performs the ARM LSL operation."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef LSR(value, amount, width):\n    if amount == 0:\n        return value\n    result, _ = LSR_C(value, amount, width)\n    return result", "response": "This function performs the ARM LSR ( logical shift right operation."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ASR_C(value, amount, width):\n    assert amount <= width\n    assert amount > 0\n    assert amount + width <= width * 2\n    value = Operators.SEXTEND(value, width, width * 2)\n    result = GetNBits(value >> amount, width)\n    carry = Bit(value, amount - 1)\n    return (result, carry)", "response": "This function performs the ARM ASR_C operation."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef ROR_C(value, amount, width):\n    assert amount <= width\n    assert amount > 0\n    m = amount % width\n    right, _ = LSR_C(value, m, width)\n    left, _ = LSL_C(value, width - m, width)\n    result = left | right\n    carry = Bit(result, width - 1)\n    return (result, carry)", "response": "The ARM ROR_C operation."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef RRX_C(value, carry, width):\n    carry_out = Bit(value, 0)\n    result = (value >> 1) | (carry << (width - 1))\n    return (result, carry_out)", "response": "The ARM RRX C operation."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef RRX(value, carry, width):\n    result, _ = RRX_C(value, carry, width)\n    return result", "response": "The ARM RRX operation."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a shared context dictionary.", "response": "def locked_context(self, key=None, default=dict):\n        \"\"\" Policy shared context dictionary \"\"\"\n        keys = ['policy']\n        if key is not None:\n            keys.append(key)\n        with self._executor.locked_context('.'.join(keys), default) as policy_context:\n            yield policy_context"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nadding a callback to be called when a state is received.", "response": "def _add_state_callback(self, state_id, state):\n        \"\"\" Save summarize(state) on policy shared context before\n            the state is stored\n        \"\"\"\n        summary = self.summarize(state)\n        if summary is None:\n            return\n        with self.locked_context('summaries', dict) as ctx:\n            ctx[state_id] = summary"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _visited_callback(self, state, pc, instr):\n        with self.locked_context('visited', set) as ctx:\n            ctx.add(pc)", "response": "Called when a PC is visited."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _visited_callback(self, state, pc, instr):\n        pc = state.platform.current.PC\n        with self.locked_context('visited', dict) as ctx:\n            ctx[pc] = ctx.get(pc, 0) + 1", "response": "Set the visited set as the current PC."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef locked_context(self, key=None, default=dict):\n        assert default in (list, dict, set)\n        with self._lock:\n            if key is None:\n                yield self._shared_context\n            else:\n                sub_context = self._shared_context.get(key, None)\n                if sub_context is None:\n                    sub_context = default()\n                yield sub_context\n                self._shared_context[key] = sub_context", "response": "A context is a shared memory object. All workers share this. A shared memory object. All workers share this. A shared memory object. All workers share this. A shared memory object. All workers share this. A shared memory object. All workers share this. A shared memory object."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the next available state with the max priority.", "response": "def get(self):\n        \"\"\" Dequeue a state with the max priority \"\"\"\n\n        # A shutdown has been requested\n        if self.is_shutdown():\n            return None\n\n        # if not more states in the queue, let's wait for some forks\n        while len(self._states) == 0:\n            # if no worker is running, bail out\n            if self.running == 0:\n                return None\n            # if a shutdown has been requested, bail out\n            if self.is_shutdown():\n                return None\n            # if there ares actually some workers running, wait for state forks\n            logger.debug(\"Waiting for available states\")\n            self._lock.wait()\n\n        state_id = self._policy.choice(list(self._states))\n        if state_id is None:\n            return None\n        del self._states[self._states.index(state_id)]\n        return state_id"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef fork(self, state, expression, policy='ALL', setstate=None):\n        assert isinstance(expression, Expression)\n\n        if setstate is None:\n            setstate = lambda x, y: None\n\n        # Find a set of solutions for expression\n        solutions = state.concretize(expression, policy)\n\n        if not solutions:\n            raise ExecutorError(\"Forking on unfeasible constraint set\")\n\n        if len(solutions) == 1:\n            setstate(state, solutions[0])\n            return state\n\n        logger.info(\"Forking. Policy: %s. Values: %s\",\n                    policy,\n                    ', '.join(f'0x{sol:x}' for sol in solutions))\n\n        self._publish('will_fork_state', state, expression, solutions, policy)\n\n        # Build and enqueue a state for each solution\n        children = []\n        for new_value in solutions:\n            with state as new_state:\n                new_state.constrain(expression == new_value)\n\n                # and set the PC of the new state to the concrete pc-dest\n                #(or other register or memory address to concrete)\n                setstate(new_state, new_value)\n\n                self._publish('did_fork_state', new_state, expression, new_value, policy)\n\n                # enqueue new_state\n                state_id = self.enqueue(new_state)\n                # maintain a list of children for logging purpose\n                children.append(state_id)\n\n        logger.info(\"Forking current state into states %r\", children)\n        return None", "response": "Forks a state on a given expression."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a new instance of a Linux state machine.", "response": "def linux(cls, path, argv=None, envp=None, entry_symbol=None, symbolic_files=None, concrete_start='', pure_symbolic=False, stdin_size=None, **kwargs):\n        \"\"\"\n        Constructor for Linux binary analysis.\n\n        :param str path: Path to binary to analyze\n        :param argv: Arguments to provide to the binary\n        :type argv: list[str]\n        :param envp: Environment to provide to the binary\n        :type envp: dict[str, str]\n        :param entry_symbol: Entry symbol to resolve to start execution\n        :type envp: str\n        :param symbolic_files: Filenames to mark as having symbolic input\n        :type symbolic_files: list[str]\n        :param str concrete_start: Concrete stdin to use before symbolic input\n        :param int stdin_size: symbolic stdin size to use\n        :param kwargs: Forwarded to the Manticore constructor\n        :return: Manticore instance, initialized with a Linux State\n        :rtype: Manticore\n        \"\"\"\n        if stdin_size is None:\n            stdin_size = consts.stdin_size\n\n        try:\n            return cls(_make_linux(path, argv, envp, entry_symbol, symbolic_files, concrete_start, pure_symbolic, stdin_size), **kwargs)\n        except elftools.common.exceptions.ELFError:\n            raise Exception(f'Invalid binary: {path}')"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds a callback to be invoked on executing a program counter. Pass None for pc to invoke callback on every instruction. Pass None for pc to invoke callback on every instruction.", "response": "def add_hook(self, pc, callback):\n        \"\"\"\n        Add a callback to be invoked on executing a program counter. Pass `None`\n        for pc to invoke callback on every instruction. `callback` should be a callable\n        that takes one :class:`~manticore.core.state.State` argument.\n\n        :param pc: Address of instruction to hook\n        :type pc: int or None\n        :param callable callback: Hook function\n        \"\"\"\n        if not (isinstance(pc, int) or pc is None):\n            raise TypeError(f\"pc must be either an int or None, not {pc.__class__.__name__}\")\n        else:\n            self._hooks.setdefault(pc, set()).add(callback)\n            if self._hooks:\n                self._executor.subscribe('will_execute_instruction', self._hook_callback)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninvoke all registered generic hooks", "response": "def _hook_callback(self, state, pc, instruction):\n        'Invoke all registered generic hooks'\n\n        # Ignore symbolic pc.\n        # TODO(yan): Should we ask the solver if any of the hooks are possible,\n        # and execute those that are?\n\n        if issymbolic(pc):\n            return\n\n        # Invoke all pc-specific hooks\n        for cb in self._hooks.get(pc, []):\n            cb(state)\n\n        # Invoke all pc-agnostic hooks\n        for cb in self._hooks.get(None, []):\n            cb(state)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngets a configuration variable group named name.", "response": "def get_group(name: str) -> _Group:\n    \"\"\"\n    Get a configuration variable group named |name|\n    \"\"\"\n    global _groups\n\n    if name in _groups:\n        return _groups[name]\n\n    group = _Group(name)\n    _groups[name] = group\n\n    return group"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsave current config state to a yml file stream identified by f", "response": "def save(f):\n    \"\"\"\n    Save current config state to an yml file stream identified by |f|\n\n    :param f: where to write the config file\n    \"\"\"\n    global _groups\n\n    c = {}\n    for group_name, group in _groups.items():\n        section = {var.name: var.value for var in group.updated_vars()}\n        if not section:\n            continue\n        c[group_name] = section\n\n    yaml.safe_dump(c, f, line_break=True)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nloads an yml - formatted configuration from file stream |f|", "response": "def parse_config(f):\n    \"\"\"\n    Load an yml-formatted configuration from file stream |f|\n\n    :param file f: Where to read the config.\n    \"\"\"\n\n    try:\n        c = yaml.safe_load(f)\n        for section_name, section in c.items():\n            group = get_group(section_name)\n\n            for key, val in section.items():\n                group.update(key)\n                setattr(group, key, val)\n    # Any exception here should trigger the warning; from not being able to parse yaml\n    # to reading poorly formatted values\n    except Exception:\n        raise ConfigError(\"Failed reading config file. Do you have a local [.]manticore.yml file?\")"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef load_overrides(path=None):\n\n    if path is not None:\n        names = [path]\n    else:\n        possible_names = ['mcore.yml', 'manticore.yml']\n        names = [os.path.join('.', ''.join(x)) for x in product(['', '.'], possible_names)]\n\n    for name in names:\n        try:\n            with open(name, 'r') as yml_f:\n                logger.info(f'Reading configuration from {name}')\n                parse_config(yml_f)\n            break\n        except FileNotFoundError:\n            pass\n    else:\n        if path is not None:\n            raise FileNotFoundError(f\"'{path}' not found for config overrides\")", "response": "Load config overrides from the yml file at path or from default paths."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef add_config_vars_to_argparse(args):\n    global _groups\n    for group_name, group in _groups.items():\n        for key in group:\n            obj = group._var_object(key)\n            args.add_argument(f\"--{group_name}.{key}\", type=type(obj.default),\n                              default=obj.default, help=obj.description)", "response": "Adds all defined config vars into args."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nbringing in provided config values to the args parser, and import entries to the config from all arguments that were actually passed on the command line :param parser: The arg parser :param args: The value that parser.parse_args returned", "response": "def process_config_values(parser: argparse.ArgumentParser, args: argparse.Namespace):\n    \"\"\"\n    Bring in provided config values to the args parser, and import entries to the config\n    from all arguments that were actually passed on the command line\n\n    :param parser: The arg parser\n    :param args: The value that parser.parse_args returned\n    \"\"\"\n    # First, load a local config file, if passed or look for one in pwd if it wasn't.\n    load_overrides(args.config)\n\n    # Get a list of defined config vals. If these are passed on the command line,\n    # update them in their correct group, not in the cli group\n    defined_vars = list(get_config_keys())\n\n    command_line_args = vars(args)\n\n    # Bring in the options keys into args\n    config_cli_args = get_group('cli')\n\n    # Place all command line args into the cli group (for saving in the workspace). If\n    # the value is set on command line, then it takes precedence; otherwise we try to\n    # read it from the config file's cli group.\n    for k in command_line_args:\n        default = parser.get_default(k)\n        set_val = getattr(args, k)\n        if default is not set_val:\n            if k not in defined_vars:\n                config_cli_args.update(k, value=set_val)\n            else:\n                # Update a var's native group\n                group_name, key = k.split('.')\n                group = get_group(group_name)\n                setattr(group, key, set_val)\n        else:\n            if k in config_cli_args:\n                setattr(args, k, getattr(config_cli_args, k))"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds a variable to this value group optionally giving it a default value and a description.", "response": "def add(self, name: str, default=None, description: str=None):\n        \"\"\"\n        Add a variable named |name| to this value group, optionally giving it a\n        default value and a description.\n\n        Variables must be added with this method before they can be set or read.\n        Reading a variable replaces the variable that was defined previously, but\n        updates the description if a new one is set.\n\n        \"\"\"\n        if name in self._vars:\n            raise ConfigError(f\"{self.name}.{name} already defined.\")\n\n        if name == 'name':\n            raise ConfigError(\"'name' is a reserved name for a group.\")\n\n        v = _Var(name, description=description, default=default)\n        self._vars[name] = v"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef update(self, name: str, value=None, default=None, description: str=None):\n        if name in self._vars:\n            description = description or self._vars[name].description\n            default = default or self._vars[name].default\n        elif name == 'name':\n            raise ConfigError(\"'name' is a reserved name for a group.\")\n\n        v = _Var(name, description=description, default=default, defined=False)\n        v.value = value\n        self._vars[name] = v", "response": "Update the value of a key in the config."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_description(self, name: str) -> str:\n        if name not in self._vars:\n            raise ConfigError(f\"{self.name}.{name} not defined.\")\n\n        return self._vars[name].description", "response": "Return the description of the variable identified by name or a help string of variable identified by name."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncommunicates with the child process without closing stdin.", "response": "def correspond(text):\n    \"\"\"Communicate with the child process without closing stdin.\"\"\"\n    subproc.stdin.write(text)\n    subproc.stdin.flush()\n    return drain()"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn the function signature for the specified name and Solidity JSON metadata inputs array.", "response": "def function_signature_for_name_and_inputs(name: str, inputs: Sequence[Mapping[str, Any]]) -> str:\n        \"\"\"Returns the function signature for the specified name and Solidity JSON metadata inputs array.\n\n        The ABI specification defines the function signature as the function name followed by the parenthesised list of\n        parameter types separated by single commas and no spaces.\n        See https://solidity.readthedocs.io/en/latest/abi-spec.html#function-selector\n        \"\"\"\n        return name + SolidityMetadata.tuple_signature_for_components(inputs)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef tuple_signature_for_components(components: Sequence[Mapping[str, Any]]) -> str:\n        ts = []\n        for c in components:\n            t: str = c['type']\n            if t.startswith('tuple'):\n                assert len(t) == 5 or t[5] == '['\n                t = SolidityMetadata.tuple_signature_for_components(c['components']) + t[5:]\n            ts.append(t)\n        return f'({\",\".join(ts)})'", "response": "Equivalent to function_signature_for_name_and_inputs ( name and inputs components )"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_constructor_arguments(self) -> str:\n        item = self._constructor_abi_item\n        return '()' if item is None else self.tuple_signature_for_components(item['inputs'])", "response": "Returns the tuple type signature for the arguments of the contract constructor."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_source_for(self, asm_offset, runtime=True):\n        srcmap = self.get_srcmap(runtime)\n\n        try:\n            beg, size, _, _ = srcmap[asm_offset]\n        except KeyError:\n            #asm_offset pointing outside the known bytecode\n            return ''\n\n        output = ''\n        nl = self.source_code[:beg].count('\\n') + 1\n        snippet = self.source_code[beg:beg + size]\n        for l in snippet.split('\\n'):\n            output += '    %s  %s\\n' % (nl, l)\n            nl += 1\n        return output", "response": "Return the source code snippet related to an arbitrary offset."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef constructor_abi(self) -> Dict[str, Any]:\n        item = self._constructor_abi_item\n        if item:\n            return dict(item)\n        return {'inputs': [], 'payable': False, 'stateMutability': 'nonpayable', 'type': 'constructor'}", "response": "Returns a copy of the Solidity JSON ABI item for the contract constructor."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a copy of the Solidity JSON ABI item for the normal contract function associated with the selector hsh.", "response": "def get_abi(self, hsh: bytes) -> Dict[str, Any]:\n        \"\"\"Returns a copy of the Solidity JSON ABI item for the function associated with the selector ``hsh``.\n\n        If no normal contract function has the specified selector, a dict describing the default or non-default\n        fallback function is returned.\n\n        The content of the returned dict is described at https://solidity.readthedocs.io/en/latest/abi-spec.html#json_\n        \"\"\"\n        if not isinstance(hsh, (bytes, bytearray)):\n            raise TypeError('The selector argument must be a concrete byte array')\n        sig = self._function_signatures_by_selector.get(hsh)\n        if sig is not None:\n            return dict(self._function_abi_items_by_signature[sig])\n        item = self._fallback_function_abi_item\n        if item is not None:\n            return dict(item)\n        # An item describing the default fallback function.\n        return {'payable': False, 'stateMutability': 'nonpayable', 'type': 'fallback'}"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the tuple type signature for the arguments of the normal contract function associated with the specified selector.", "response": "def get_func_argument_types(self, hsh: bytes):\n        \"\"\"Returns the tuple type signature for the arguments of the function associated with the selector ``hsh``.\n\n        If no normal contract function has the specified selector,\n        the empty tuple type signature ``'()'`` is returned.\n        \"\"\"\n        if not isinstance(hsh, (bytes, bytearray)):\n            raise TypeError('The selector argument must be a concrete byte array')\n        sig = self._function_signatures_by_selector.get(hsh)\n        return '()' if sig is None else sig[sig.find('('):]"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_func_return_types(self, hsh: bytes) -> str:\n        if not isinstance(hsh, (bytes, bytearray)):\n            raise TypeError('The selector argument must be a concrete byte array')\n        abi = self.get_abi(hsh)\n        outputs = abi.get('outputs')\n        return '()' if outputs is None else SolidityMetadata.tuple_signature_for_components(outputs)", "response": "Returns the tuple type signature for the output values of the normal contract function with the specified selector."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_func_name(self, hsh: bytes) -> str:\n        if not isinstance(hsh, (bytes, bytearray)):\n            raise TypeError('The selector argument must be a concrete byte array')\n        sig = self._function_signatures_by_selector.get(hsh)\n        return '{fallback}' if sig is None else sig[:sig.find('(')]", "response": "Returns the name of the normal function with the selector hsh."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_func_signature(self, hsh: bytes) -> Optional[str]:\n        if not isinstance(hsh, (bytes, bytearray)):\n            raise TypeError('The selector argument must be a concrete byte array')\n        return self._function_signatures_by_selector.get(hsh)", "response": "Returns the signature of the normal function with the selector hsh or None if no such function exists."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef convert_permissions(m_perms):\n    permissions = UC_PROT_NONE\n    if 'r' in m_perms:\n        permissions |= UC_PROT_READ\n    if 'w' in m_perms:\n        permissions |= UC_PROT_WRITE\n    if 'x' in m_perms:\n        permissions |= UC_PROT_EXEC\n    return permissions", "response": "Converts a Manticore permission string into a Unicorn permission string."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef copy_memory(self, address, size):\n        start_time = time.time()\n        map_bytes = self._cpu._raw_read(address, size)\n        self._emu.mem_write(address, map_bytes)\n        if time.time() - start_time > 3:\n            logger.info(f\"Copying {hr_size(size)} map at {hex(address)} took {time.time() - start_time} seconds\")", "response": "Copy the bytes from address to address + size into Unicorn\n        Used primarily for copying memory maps\n           "}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef map_memory_callback(self, address, size, perms, name, offset, result):\n        logger.info(' '.join((\"Mapping Memory @\",\n                              hex(address) if type(address) is int else \"0x??\",\n                              hr_size(size), \"-\",\n                              perms, \"-\",\n                              f\"{name}:{hex(offset) if name else ''}\", \"->\",\n                              hex(result))))\n        self._emu.mem_map(address, size, convert_permissions(perms))\n        self.copy_memory(address, size)", "response": "Catches did_map_memory and copies the mapping into Manticore\n       "}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef protect_memory_callback(self, start, size, perms):\n        logger.info(f\"Changing permissions on {hex(start)}:{hex(start + size)} to {perms}\")\n        self._emu.mem_protect(start, size, convert_permissions(perms))", "response": "Set memory protections in Unicorn correctly"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _hook_syscall(self, uc, data):\n        logger.debug(f\"Stopping emulation at {hex(uc.reg_read(self._to_unicorn_id('RIP')))} to perform syscall\")\n        self.sync_unicorn_to_manticore()\n        from ..native.cpu.abstractcpu import Syscall\n        self._to_raise = Syscall()\n        uc.emu_stop()", "response": "Hook that transfers control to Manticore so it can execute the syscall"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _hook_write_mem(self, uc, access, address, size, value, data):\n        self._mem_delta[address] = (value, size)\n        return True", "response": "Hook for writing memory to Unicorn."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nhook for unmapped memory.", "response": "def _hook_unmapped(self, uc, access, address, size, value, data):\n        \"\"\"\n        We hit an unmapped region; map it into unicorn.\n        \"\"\"\n        try:\n            self.sync_unicorn_to_manticore()\n            logger.warning(f\"Encountered an operation on unmapped memory at {hex(address)}\")\n            m = self._cpu.memory.map_containing(address)\n            self.copy_memory(m.start, m.end - m.start)\n        except MemoryException as e:\n            logger.error(\"Failed to map memory {}-{}, ({}): {}\".format(hex(address), hex(address + size), access, e))\n            self._to_raise = e\n            self._should_try_again = False\n            return False\n\n        self._should_try_again = True\n        return False"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef emulate(self, instruction):\n\n        # The emulation might restart if Unicorn needs to bring in a memory map\n        # or bring a value from Manticore state.\n        while True:\n\n            # Try emulation\n            self._should_try_again = False\n            self._to_raise = None\n\n            self._step(instruction)\n\n            if not self._should_try_again:\n                break", "response": "Wrapper that emulates the internal state of the internal state."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _step(self, instruction, chunksize=0):\n\n        try:\n            pc = self._cpu.PC\n            m = self._cpu.memory.map_containing(pc)\n            if self._stop_at:\n                logger.info(f\"Emulating from {hex(pc)} to  {hex(self._stop_at)}\")\n            self._emu.emu_start(pc, m.end if not self._stop_at else self._stop_at, count=chunksize)\n        except UcError:\n            # We request re-execution by signaling error; if we we didn't set\n            # _should_try_again, it was likely an actual error\n            if not self._should_try_again:\n                raise\n\n        if self._should_try_again:\n            return\n\n        # self.sync_unicorn_to_manticore()\n        self._cpu.PC = self.get_unicorn_pc()\n        if self._cpu.PC == self._stop_at:\n            logger.info(\"Reached emulation target, switching to Manticore mode\")\n            self.sync_unicorn_to_manticore()\n            self._stop_at = None\n\n        # Raise the exception from a hook that Unicorn would have eaten\n        if self._to_raise:\n            from ..native.cpu.abstractcpu import Syscall\n            if type(self._to_raise) is not Syscall:\n                logger.info(\"Raising %s\", self._to_raise)\n            raise self._to_raise\n\n        logger.info(f\"Exiting Unicorn Mode at {hex(self._cpu.PC)}\")\n        return", "response": "Execute a chunk of instructions starting from instruction\n           ."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncopies registers and written memory back into Manticore", "response": "def sync_unicorn_to_manticore(self):\n        \"\"\"\n        Copy registers and written memory back into Manticore\n        \"\"\"\n        self.write_backs_disabled = True\n        for reg in self.registers:\n            val = self._emu.reg_read(self._to_unicorn_id(reg))\n            self._cpu.write_register(reg, val)\n        if len(self._mem_delta) > 0:\n            logger.debug(f\"Syncing {len(self._mem_delta)} writes back into Manticore\")\n        for location in self._mem_delta:\n            value, size = self._mem_delta[location]\n            self._cpu.write_int(location, value, size * 8)\n        self.write_backs_disabled = False\n        self._mem_delta = {}"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncopying memory writes from Manticore back into Unicorn in real - time", "response": "def write_back_memory(self, where, expr, size):\n        \"\"\" Copy memory writes from Manticore back into Unicorn in real-time \"\"\"\n        if self.write_backs_disabled:\n            return\n        if type(expr) is bytes:\n            self._emu.mem_write(where, expr)\n        else:\n            if issymbolic(expr):\n                data = [Operators.CHR(Operators.EXTRACT(expr, offset, 8)) for offset in range(0, size, 8)]\n                concrete_data = []\n                for c in data:\n                    if issymbolic(c):\n                        c = chr(solver.get_value(self._cpu.memory.constraints, c))\n                    concrete_data.append(c)\n                data = concrete_data\n            else:\n                data = [Operators.CHR(Operators.EXTRACT(expr, offset, 8)) for offset in range(0, size, 8)]\n            logger.debug(f\"Writing back {hr_size(size // 8)} to {hex(where)}: {data}\")\n            # TODO - the extra encoding is to handle null bytes output as strings when we concretize. That's probably a bug.\n            self._emu.mem_write(where, b''.join(b.encode('utf-8') if type(b) is str else b for b in data))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef write_back_register(self, reg, val):\n        if self.write_backs_disabled:\n            return\n        if issymbolic(val):\n            logger.warning(\"Skipping Symbolic write-back\")\n            return\n        if reg in self.flag_registers:\n            self._emu.reg_write(self._to_unicorn_id('EFLAGS'), self._cpu.read_register('EFLAGS'))\n            return\n        self._emu.reg_write(self._to_unicorn_id(reg), val)", "response": "Write back a register."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update_segment(self, selector, base, size, perms):\n        logger.info(\"Updating selector %s to 0x%02x (%s bytes) (%s)\", selector, base, size, perms)\n        if selector == 99:\n            self.set_fs(base)\n        else:\n            logger.error(\"No way to write segment: %d\", selector)", "response": "Update the internal state of the current segment."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef flip(constraint):\n    '''\n    flips a constraint (Equal)\n\n    (Equal (BitVecITE Cond IfC ElseC) IfC)\n        ->\n    (Equal (BitVecITE Cond IfC ElseC) ElseC)\n    '''\n    equal = copy.copy(constraint)\n\n    assert len(equal.operands) == 2\n    # assume they are the equal -> ite form that we produce on standard branches\n    ite, forcepc = equal.operands\n    assert isinstance(ite, BitVecITE) and isinstance(forcepc, BitVecConstant)\n    assert len(ite.operands) == 3\n    cond, iifpc, eelsepc = ite.operands\n    assert isinstance(iifpc, BitVecConstant) and isinstance(eelsepc, BitVecConstant)\n\n    equal._operands= (equal.operands[0], eelsepc if forcepc.value == iifpc.value else iifpc)\n\n    return equal", "response": "Flips a constraint into a single node."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ngenerate permutations of lst where permutations are mutated by func. Used for flipping constraints.", "response": "def perm(lst, func):\n    ''' Produce permutations of `lst`, where permutations are mutated by `func`. Used for flipping constraints. highly\n    possible that returned constraints can be unsat this does it blindly, without any attention to the constraints\n    themselves\n\n    Considering lst as a list of constraints, e.g.\n\n      [ C1, C2, C3 ]\n\n    we'd like to consider scenarios of all possible permutations of flipped constraints, excluding the original list.\n    So we'd like to generate:\n\n      [ func(C1),      C2 ,       C3 ],\n      [      C1 , func(C2),       C3 ],\n      [ func(C1), func(C2),       C3 ],\n      [      C1 ,      C2 ,  func(C3)],\n      .. etc\n\n    This is effectively treating the list of constraints as a bitmask of width len(lst) and counting up, skipping the\n    0th element (unmodified array).\n\n    The code below yields lists of constraints permuted as above by treating list indeces as bitmasks from 1 to\n     2**len(lst) and applying func to all the set bit offsets.\n\n    '''\n    for i in range(1, 2**len(lst)):\n        yield [func(item) if (1<<j)&i else item for (j, item) in enumerate(lst)]"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef input_from_cons(constupl, datas):\n    ' solve bytes in |datas| based on '\n    def make_chr(c):\n        try:\n            return chr(c)\n        except Exception:\n            return c\n    newset = constraints_to_constraintset(constupl)\n\n    ret = ''\n    for data in datas:\n        for c in data:\n            ret += make_chr(solver.get_value(newset, c))\n    return ret", "response": "solve bytes in |datas| based on '\n   "}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nexecuting a symbolic run that follows a concrete run ; return constraints generated and the stdin data produced", "response": "def symbolic_run_get_cons(trace):\n    '''\n    Execute a symbolic run that follows a concrete run; return constraints generated\n    and the stdin data produced\n    '''\n\n    m2 = Manticore.linux(prog, workspace_url='mem:')\n    f = Follower(trace)\n    m2.verbosity(VERBOSITY)\n    m2.register_plugin(f)\n\n    def on_term_testcase(mcore, state, stateid, err):\n        with m2.locked_context() as ctx:\n            readdata = []\n            for name, fd, data in state.platform.syscall_trace:\n                if name in ('_receive', '_read') and fd == 0:\n                    readdata.append(data)\n            ctx['readdata'] = readdata\n            ctx['constraints'] = list(state.constraints.constraints)\n\n    m2.subscribe('will_terminate_state', on_term_testcase)\n\n    m2.run()\n\n    constraints = m2.context['constraints']\n    datas = m2.context['readdata']\n\n    return constraints, datas"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nmove the file to the specified offset according to whence.", "response": "def seek(self, offset, whence=os.SEEK_SET):\n        \"\"\"\n        Repositions the file C{offset} according to C{whence}.\n        Returns the resulting offset or -1 in case of error.\n        :rtype: int\n        :return: the file offset.\n        \"\"\"\n        assert isinstance(offset, int)\n        assert whence in (os.SEEK_SET, os.SEEK_CUR, os.SEEK_END)\n\n        new_position = 0\n        if whence == os.SEEK_SET:\n            new_position = offset\n        elif whence == os.SEEK_CUR:\n            new_position = self.pos + offset\n        elif whence == os.SEEK_END:\n            new_position = self.max_size + offset\n\n        if new_position < 0:\n            return -1\n\n        self.pos = new_position\n\n        return self.pos"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreading up to count bytes from the file.", "response": "def read(self, count):\n        \"\"\"\n        Reads up to C{count} bytes from the file.\n        :rtype: list\n        :return: the list of symbolic bytes read\n        \"\"\"\n        if self.pos > self.max_size:\n            return []\n        else:\n            size = min(count, self.max_size - self.pos)\n            ret = [self.array[i] for i in range(self.pos, self.pos + size)]\n            self.pos += size\n            return ret"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef write(self, data):\n        size = min(len(data), self.max_size - self.pos)\n        for i in range(self.pos, self.pos + size):\n            self.array[i] = data[i - self.pos]", "response": "Writes the symbolic bytes in data onto the file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate a new Linux object with an empty CPU and standard file set.", "response": "def empty_platform(cls, arch):\n        \"\"\"\n        Create a platform without an ELF loaded.\n\n        :param str arch: The architecture of the new platform\n        :rtype: Linux\n        \"\"\"\n        platform = cls(None)\n        platform._init_cpu(arch)\n        platform._init_std_fds()\n        return platform"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nload and establish program state.", "response": "def _execve(self, program, argv, envp):\n        \"\"\"\n        Load `program` and establish program state, such as stack and arguments.\n\n        :param program str: The ELF binary to load\n        :param argv list: argv array\n        :param envp list: envp array\n        \"\"\"\n        argv = [] if argv is None else argv\n        envp = [] if envp is None else envp\n\n        logger.debug(f\"Loading {program} as a {self.arch} elf\")\n\n        self.load(program, envp)\n        self._arch_specific_init()\n\n        self._stack_top = self.current.STACK\n        self.setup_stack([program] + argv, envp)\n\n        nprocs = len(self.procs)\n        nfiles = len(self.files)\n        assert nprocs > 0\n        self.running = list(range(nprocs))\n\n        # Each process can wait for one timeout\n        self.timers = [None] * nprocs\n        # each fd has a waitlist\n        self.rwait = [set() for _ in range(nfiles)]\n        self.twait = [set() for _ in range(nfiles)]\n\n        # Install event forwarders\n        for proc in self.procs:\n            self.forward_events_from(proc)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninitialize the ARM kernel helpers for the current ARM kernel.", "response": "def _init_arm_kernel_helpers(self):\n        \"\"\"\n        ARM kernel helpers\n\n        https://www.kernel.org/doc/Documentation/arm/kernel_user_helpers.txt\n        \"\"\"\n\n        page_data = bytearray(b'\\xf1\\xde\\xfd\\xe7' * 1024)\n\n        # Extracted from a RPi2\n        preamble = binascii.unhexlify(\n            'ff0300ea' +\n            '650400ea' +\n            'f0ff9fe5' +\n            '430400ea' +\n            '220400ea' +\n            '810400ea' +\n            '000400ea' +\n            '870400ea'\n        )\n\n        # XXX(yan): The following implementations of cmpxchg and cmpxchg64 were\n        # handwritten to not use any exclusive instructions (e.g. ldrexd) or\n        # locking. For actual implementations, refer to\n        # arch/arm64/kernel/kuser32.S in the Linux source code.\n        __kuser_cmpxchg64 = binascii.unhexlify(\n            '30002de9' +  # push    {r4, r5}\n            '08c09de5' +  # ldr     ip, [sp, #8]\n            '30009ce8' +  # ldm     ip, {r4, r5}\n            '010055e1' +  # cmp     r5, r1\n            '00005401' +  # cmpeq   r4, r0\n            '0100a013' +  # movne   r0, #1\n            '0000a003' +  # moveq   r0, #0\n            '0c008c08' +  # stmeq   ip, {r2, r3}\n            '3000bde8' +  # pop     {r4, r5}\n            '1eff2fe1'   # bx      lr\n        )\n\n        __kuser_dmb = binascii.unhexlify(\n            '5bf07ff5' +  # dmb ish\n            '1eff2fe1'   # bx lr\n        )\n\n        __kuser_cmpxchg = binascii.unhexlify(\n            '003092e5' +  # ldr     r3, [r2]\n            '000053e1' +  # cmp     r3, r0\n            '0000a003' +  # moveq   r0, #0\n            '00108205' +  # streq   r1, [r2]\n            '0100a013' +  # movne   r0, #1\n            '1eff2fe1'   # bx      lr\n        )\n\n        # Map a TLS segment\n        self._arm_tls_memory = self.current.memory.mmap(None, 4, 'rw ')\n\n        __kuser_get_tls = binascii.unhexlify(\n            '04009FE5' +  # ldr r0, [pc, #4]\n            '010090e8' +  # ldm r0, {r0}\n            '1eff2fe1'   # bx lr\n        ) + struct.pack('<I', self._arm_tls_memory)\n\n        tls_area = b'\\x00' * 12\n\n        version = struct.pack('<I', 5)\n\n        def update(address, code):\n            page_data[address:address + len(code)] = code\n\n        # Offsets from Documentation/arm/kernel_user_helpers.txt in Linux\n        update(0x000, preamble)\n        update(0xf60, __kuser_cmpxchg64)\n        update(0xfa0, __kuser_dmb)\n        update(0xfc0, __kuser_cmpxchg)\n        update(0xfe0, __kuser_get_tls)\n        update(0xff0, tls_area)\n        update(0xffc, version)\n\n        self.current.memory.mmap(0xffff0000, len(page_data), 'r x', page_data)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef setup_stack(self, argv, envp):\n        cpu = self.current\n\n        # In case setup_stack() is called again, we make sure we're growing the\n        # stack from the original top\n        cpu.STACK = self._stack_top\n\n        auxv = self.auxv\n        logger.debug(\"Setting argv, envp and auxv.\")\n        logger.debug(f\"\\tArguments: {argv!r}\")\n        if envp:\n            logger.debug(\"\\tEnvironment:\")\n            for e in envp:\n                logger.debug(f\"\\t\\t{e!r}\")\n\n        logger.debug(\"\\tAuxv:\")\n        for name, val in auxv.items():\n            logger.debug(f\"\\t\\t{name}: 0x{val:x}\")\n\n        # We save the argument and environment pointers\n        argvlst = []\n        envplst = []\n\n        # end envp marker empty string\n        for evar in envp:\n            cpu.push_bytes('\\x00')\n            envplst.append(cpu.push_bytes(evar))\n\n        for arg in argv:\n            cpu.push_bytes('\\x00')\n            argvlst.append(cpu.push_bytes(arg))\n\n        # Put all auxv strings into the string stack area.\n        # And replace the value be its pointer\n\n        for name, value in auxv.items():\n            if hasattr(value, '__len__'):\n                cpu.push_bytes(value)\n                auxv[name] = cpu.STACK\n\n        # The \"secure execution\" mode of secure_getenv() is controlled by the\n        # AT_SECURE flag contained in the auxiliary vector passed from the\n        # kernel to user space.\n        auxvnames = {\n            'AT_IGNORE': 1,  # Entry should be ignored\n            'AT_EXECFD': 2,  # File descriptor of program\n            'AT_PHDR': 3,  # Program headers for program\n            'AT_PHENT': 4,  # Size of program header entry\n            'AT_PHNUM': 5,  # Number of program headers\n            'AT_PAGESZ': 6,  # System page size\n            'AT_BASE': 7,  # Base address of interpreter\n            'AT_FLAGS': 8,  # Flags\n            'AT_ENTRY': 9,  # Entry point of program\n            'AT_NOTELF': 10,  # Program is not ELF\n            'AT_UID': 11,  # Real uid\n            'AT_EUID': 12,  # Effective uid\n            'AT_GID': 13,  # Real gid\n            'AT_EGID': 14,  # Effective gid\n            'AT_CLKTCK': 17,  # Frequency of times()\n            'AT_PLATFORM': 15,  # String identifying platform.\n            'AT_HWCAP': 16,  # Machine-dependent hints about processor capabilities.\n            'AT_FPUCW': 18,  # Used FPU control word.\n            'AT_SECURE': 23,  # Boolean, was exec setuid-like?\n            'AT_BASE_PLATFORM': 24,  # String identifying real platforms.\n            'AT_RANDOM': 25,  # Address of 16 random bytes.\n            'AT_EXECFN': 31,  # Filename of executable.\n            'AT_SYSINFO': 32,  # Pointer to the global system page used for system calls and other nice things.\n            'AT_SYSINFO_EHDR': 33,  # Pointer to the global system page used for system calls and other nice things.\n        }\n        # AT_NULL\n        cpu.push_int(0)\n        cpu.push_int(0)\n        for name, val in auxv.items():\n            cpu.push_int(val)\n            cpu.push_int(auxvnames[name])\n\n        # NULL ENVP\n        cpu.push_int(0)\n        for var in reversed(envplst):              # ENVP n\n            cpu.push_int(var)\n        envp = cpu.STACK\n\n        # NULL ARGV\n        cpu.push_int(0)\n        for arg in reversed(argvlst):              # Argv n\n            cpu.push_int(arg)\n        argv = cpu.STACK\n\n        # ARGC\n        cpu.push_int(len(argvlst))", "response": "This function is called by the parser to set up the stack."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nload and prepares an ELF program in memory and prepares the initial CPU state.", "response": "def load(self, filename, env):\n        \"\"\"\n        Loads and an ELF program in memory and prepares the initial CPU state.\n        Creates the stack and loads the environment variables and the arguments in it.\n\n        :param filename: pathname of the file to be executed. (used for auxv)\n        :param list env: A list of env variables. (used for extracting vars that control ld behavior)\n        :raises error:\n            - 'Not matching cpu': if the program is compiled for a different architecture\n            - 'Not matching memory': if the program is compiled for a different address size\n        :todo: define va_randomize and read_implies_exec personality\n        \"\"\"\n        # load elf See binfmt_elf.c\n        # read the ELF object file\n        cpu = self.current\n        elf = self.elf\n        arch = self.arch\n        env = dict(var.split('=') for var in env if '=' in var)\n        addressbitsize = {'x86': 32, 'x64': 64, 'ARM': 32}[elf.get_machine_arch()]\n        logger.debug(\"Loading %s as a %s elf\", filename, arch)\n\n        assert elf.header.e_type in ['ET_DYN', 'ET_EXEC', 'ET_CORE']\n\n        # Get interpreter elf\n        interpreter = None\n        for elf_segment in elf.iter_segments():\n            if elf_segment.header.p_type != 'PT_INTERP':\n                continue\n            interpreter_filename = elf_segment.data()[:-1]\n            logger.info(f'Interpreter filename: {interpreter_filename}')\n            if os.path.exists(interpreter_filename.decode('utf-8')):\n                interpreter = ELFFile(open(interpreter_filename, 'rb'))\n            elif 'LD_LIBRARY_PATH' in env:\n                for mpath in env['LD_LIBRARY_PATH'].split(\":\"):\n                    interpreter_path_filename = os.path.join(mpath, os.path.basename(interpreter_filename))\n                    logger.info(f\"looking for interpreter {interpreter_path_filename}\")\n                    if os.path.exists(interpreter_path_filename):\n                        interpreter = ELFFile(open(interpreter_path_filename, 'rb'))\n                        break\n            break\n        if interpreter is not None:\n            assert interpreter.get_machine_arch() == elf.get_machine_arch()\n            assert interpreter.header.e_type in ['ET_DYN', 'ET_EXEC']\n\n        # Stack Executability\n        executable_stack = False\n        for elf_segment in elf.iter_segments():\n            if elf_segment.header.p_type != 'PT_GNU_STACK':\n                continue\n            if elf_segment.header.p_flags & 0x01:\n                executable_stack = True\n            else:\n                executable_stack = False\n            break\n\n        base = 0\n        elf_bss = 0\n        end_code = 0\n        end_data = 0\n        elf_brk = 0\n        self.load_addr = 0\n\n        for elf_segment in elf.iter_segments():\n            if elf_segment.header.p_type != 'PT_LOAD':\n                continue\n\n            align = 0x1000  # elf_segment.header.p_align\n\n            ELF_PAGEOFFSET = elf_segment.header.p_vaddr & (align - 1)\n\n            flags = elf_segment.header.p_flags\n            memsz = elf_segment.header.p_memsz + ELF_PAGEOFFSET\n            offset = elf_segment.header.p_offset - ELF_PAGEOFFSET\n            filesz = elf_segment.header.p_filesz + ELF_PAGEOFFSET\n            vaddr = elf_segment.header.p_vaddr - ELF_PAGEOFFSET\n            memsz = cpu.memory._ceil(memsz)\n            if base == 0 and elf.header.e_type == 'ET_DYN':\n                assert vaddr == 0\n                if addressbitsize == 32:\n                    base = 0x56555000\n                else:\n                    base = 0x555555554000\n\n            perms = perms_from_elf(flags)\n            hint = base + vaddr\n            if hint == 0:\n                hint = None\n\n            logger.debug(f\"Loading elf offset: {offset:08x} addr:{base + vaddr:08x} {base + vaddr + memsz:08x} {perms}\")\n            base = cpu.memory.mmapFile(hint, memsz, perms, elf_segment.stream.name, offset) - vaddr\n\n            if self.load_addr == 0:\n                self.load_addr = base + vaddr\n\n            k = base + vaddr + filesz\n            if k > elf_bss:\n                elf_bss = k\n            if (flags & 4) and end_code < k:  # PF_X\n                end_code = k\n            if end_data < k:\n                end_data = k\n            k = base + vaddr + memsz\n            if k > elf_brk:\n                elf_brk = k\n\n        elf_entry = elf.header.e_entry\n        if elf.header.e_type == 'ET_DYN':\n            elf_entry += self.load_addr\n        entry = elf_entry\n        real_elf_brk = elf_brk\n\n        # We need to explicitly clear bss, as fractional pages will have data from the file\n        bytes_to_clear = elf_brk - elf_bss\n        if bytes_to_clear > 0:\n            logger.debug(f\"Zeroing main elf fractional pages. From bss({elf_bss:x}) to brk({elf_brk:x}), {bytes_to_clear} bytes.\")\n            cpu.write_bytes(elf_bss, '\\x00' * bytes_to_clear, force=True)\n\n        stack_size = 0x21000\n\n        if addressbitsize == 32:\n            stack_top = 0xc0000000\n        else:\n            stack_top = 0x800000000000\n        stack_base = stack_top - stack_size\n        stack = cpu.memory.mmap(stack_base, stack_size, 'rwx', name='stack') + stack_size\n        assert stack_top == stack\n\n        reserved = cpu.memory.mmap(base + vaddr + memsz, 0x1000000, '   ')\n        interpreter_base = 0\n        if interpreter is not None:\n            base = 0\n            elf_bss = 0\n            end_code = 0\n            end_data = 0\n            elf_brk = 0\n            entry = interpreter.header.e_entry\n            for elf_segment in interpreter.iter_segments():\n                if elf_segment.header.p_type != 'PT_LOAD':\n                    continue\n                align = 0x1000  # elf_segment.header.p_align\n                vaddr = elf_segment.header.p_vaddr\n                filesz = elf_segment.header.p_filesz\n                flags = elf_segment.header.p_flags\n                offset = elf_segment.header.p_offset\n                memsz = elf_segment.header.p_memsz\n\n                ELF_PAGEOFFSET = (vaddr & (align - 1))\n                memsz = memsz + ELF_PAGEOFFSET\n                offset = offset - ELF_PAGEOFFSET\n                filesz = filesz + ELF_PAGEOFFSET\n                vaddr = vaddr - ELF_PAGEOFFSET\n                memsz = cpu.memory._ceil(memsz)\n\n                if base == 0 and interpreter.header.e_type == 'ET_DYN':\n                    assert vaddr == 0\n                    total_size = self._interp_total_size(interpreter)\n                    base = stack_base - total_size\n\n                if base == 0:\n                    assert vaddr == 0\n                perms = perms_from_elf(flags)\n                hint = base + vaddr\n                if hint == 0:\n                    hint = None\n\n                base = cpu.memory.mmapFile(hint, memsz, perms, elf_segment.stream.name, offset)\n                base -= vaddr\n                logger.debug(\n                    f\"Loading interpreter offset: {offset:08x} \"\n                    f\"addr:{base + vaddr:08x} \"\n                    f\"{base + vaddr + memsz:08x} \"\n                    f\"{(flags & 1 and 'r' or ' ')}\"\n                    f\"{(flags & 2 and 'w' or ' ')}\"\n                    f\"{(flags & 4 and 'x' or ' ')}\"\n                )\n\n                k = base + vaddr + filesz\n                if k > elf_bss:\n                    elf_bss = k\n                if (flags & 4) and end_code < k:  # PF_X\n                    end_code = k\n                if end_data < k:\n                    end_data = k\n                k = base + vaddr + memsz\n                if k > elf_brk:\n                    elf_brk = k\n\n            if interpreter.header.e_type == 'ET_DYN':\n                entry += base\n            interpreter_base = base\n\n            bytes_to_clear = elf_brk - elf_bss\n            if bytes_to_clear > 0:\n                logger.debug(f\"Zeroing interpreter elf fractional pages. From bss({elf_bss:x}) to brk({elf_brk:x}), {bytes_to_clear} bytes.\")\n                cpu.write_bytes(elf_bss, '\\x00' * bytes_to_clear, force=True)\n\n        # free reserved brk space\n        cpu.memory.munmap(reserved, 0x1000000)\n\n        # load vdso\n        #vdso_addr = load_vdso(addressbitsize)\n\n        cpu.STACK = stack\n        cpu.PC = entry\n\n        logger.debug(f\"Entry point: {entry:016x}\")\n        logger.debug(f\"Stack start: {stack:016x}\")\n        logger.debug(f\"Brk: {real_elf_brk:016x}\")\n        logger.debug(f\"Mappings:\")\n        for m in str(cpu.memory).split('\\n'):\n            logger.debug(f\"  {m}\")\n        self.base = base\n        self.elf_bss = elf_bss\n        self.end_code = end_code\n        self.end_data = end_data\n        self.elf_brk = real_elf_brk\n        self.brk = real_elf_brk\n\n        at_random = cpu.push_bytes('A' * 16)\n        at_execfn = cpu.push_bytes(f'{filename}\\x00')\n\n        self.auxv = {\n            'AT_PHDR': self.load_addr + elf.header.e_phoff,  # Program headers for program\n            'AT_PHENT': elf.header.e_phentsize,       # Size of program header entry\n            'AT_PHNUM': elf.header.e_phnum,           # Number of program headers\n            'AT_PAGESZ': cpu.memory.page_size,         # System page size\n            'AT_BASE': interpreter_base,             # Base address of interpreter\n            'AT_FLAGS': elf.header.e_flags,           # Flags\n            'AT_ENTRY': elf_entry,                    # Entry point of program\n            'AT_UID': 1000,                         # Real uid\n            'AT_EUID': 1000,                         # Effective uid\n            'AT_GID': 1000,                         # Real gid\n            'AT_EGID': 1000,                         # Effective gid\n            'AT_CLKTCK': 100,                          # Frequency of times()\n            'AT_HWCAP': 0,                            # Machine-dependent hints about processor capabilities.\n            'AT_RANDOM': at_random,                    # Address of 16 random bytes.\n            'AT_EXECFN': at_execfn,                    # Filename of executable.\n        }"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nadding a file descriptor to the current file descriptor list. Returns the index of the file descriptor in the file descr.", "response": "def _open(self, f):\n        \"\"\"\n        Adds a file descriptor to the current file descriptor list\n\n        :rtype: int\n        :param f: the file descriptor to add.\n        :return: the index of the file descriptor in the file descr. list\n        \"\"\"\n        if None in self.files:\n            fd = self.files.index(None)\n            self.files[fd] = f\n        else:\n            fd = len(self.files)\n            self.files.append(f)\n        return fd"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nclose a file descriptor.", "response": "def _close(self, fd):\n        \"\"\"\n        Removes a file descriptor from the file descriptor list\n        :rtype: int\n        :param fd: the file descriptor to close.\n        :return: C{0} on success.\n        \"\"\"\n        try:\n            self.files[fd].close()\n            self._closed_files.append(self.files[fd])  # Keep track for SymbolicFile testcase generation\n            self.files[fd] = None\n        except IndexError:\n            raise FdError(f\"Bad file descriptor ({fd})\")"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _is_fd_open(self, fd):\n        return fd >= 0 and fd < len(self.files) and self.files[fd] is not None", "response": "Determines if the file descriptor is open."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef sys_chdir(self, path):\n        path_str = self.current.read_string(path)\n        logger.debug(f\"chdir({path_str})\")\n        try:\n            os.chdir(path_str)\n            return 0\n        except OSError as e:\n            return e.errno", "response": "Change current working directory"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef sys_getcwd(self, buf, size):\n\n        try:\n            current_dir = os.getcwd()\n            length = len(current_dir) + 1\n\n            if size > 0 and size < length:\n                logger.info(\"GETCWD: size is greater than 0, but is smaller than the length\"\n                            \"of the path + 1. Returning ERANGE\")\n                return -errno.ERANGE\n\n            if not self.current.memory.access_ok(slice(buf, buf + length), 'w'):\n                logger.info(\"GETCWD: buf within invalid memory. Returning EFAULT\")\n                return -errno.EFAULT\n\n            self.current.write_string(buf, current_dir)\n            logger.debug(f\"getcwd(0x{buf:08x}, {size}) -> <{current_dir}> (Size {length})\")\n            return length\n\n        except OSError as e:\n            return -e.errno", "response": "Get the current working directory and return the size of the array."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrite - send bytes through a file descriptor The write system call writes up to count bytes from the buffer pointed to by buf to the file descriptor fd. If count is zero, write returns 0 and optionally sets *tx_bytes to zero. :param fd a valid file descriptor :param buf a memory buffer :param count number of bytes to send :return: 0 Success EBADF fd is not a valid file descriptor or is not open. EFAULT buf or tx_bytes points to an invalid address.", "response": "def sys_write(self, fd, buf, count):\n        \"\"\" write - send bytes through a file descriptor\n          The write system call writes up to count bytes from the buffer pointed\n          to by buf to the file descriptor fd. If count is zero, write returns 0\n          and optionally sets *tx_bytes to zero.\n\n          :param fd            a valid file descriptor\n          :param buf           a memory buffer\n          :param count         number of bytes to send\n          :return: 0          Success\n                    EBADF      fd is not a valid file descriptor or is not open.\n                    EFAULT     buf or tx_bytes points to an invalid address.\n        \"\"\"\n        data: bytes = bytes()\n        cpu = self.current\n        if count != 0:\n            try:\n                write_fd = self._get_fd(fd)\n            except FdError as e:\n                logger.error(f\"WRITE: Not valid file descriptor ({fd}). Returning -{e.err}\")\n                return -e.err\n\n            # TODO check count bytes from buf\n            if buf not in cpu.memory or buf + count not in cpu.memory:\n                logger.debug(\"WRITE: buf points to invalid address. Returning EFAULT\")\n                return -errno.EFAULT\n\n            if fd > 2 and write_fd.is_full():\n                cpu.PC -= cpu.instruction.size\n                self.wait([], [fd], None)\n                raise RestartSyscall()\n\n            data: MixedSymbolicBuffer = cpu.read_bytes(buf, count)\n            data: bytes = self._transform_write_data(data)\n            write_fd.write(data)\n\n            for line in data.split(b'\\n'):\n                line = line.decode('latin-1')  # latin-1 encoding will happily decode any byte (0x00-0xff)\n                logger.debug(f\"WRITE({fd}, 0x{buf:08x}, {count}) -> <{repr(line):48s}>\")\n            self.syscall_trace.append((\"_write\", fd, data))\n            self.signal_transmit(fd)\n\n        return len(data)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef sys_access(self, buf, mode):\n        filename = b''\n        for i in range(0, 255):\n            c = Operators.CHR(self.current.read_int(buf + i, 8))\n            if c == b'\\x00':\n                break\n            filename += c\n\n        if os.access(filename, mode):\n            return 0\n        else:\n            return -1", "response": "Checks real user s permissions for a file."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef sys_newuname(self, old_utsname):\n        from datetime import datetime\n\n        def pad(s):\n            return s + '\\x00' * (65 - len(s))\n\n        now = datetime(2017, 8, 0o1).strftime(\"%a %b %d %H:%M:%S ART %Y\")\n        info = (('sysname', 'Linux'),\n                ('nodename', 'ubuntu'),\n                ('release', '4.4.0-77-generic'),\n                ('version', '#98 SMP ' + now),\n                ('machine', self._uname_machine),\n                ('domainname', ''))\n\n        uname_buf = ''.join(pad(pair[1]) for pair in info)\n        self.current.write_bytes(old_utsname, uname_buf)\n        return 0", "response": "Writes system information in the variable old_utsname."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nchange data segment size (moves the C{brk} to the new address) :rtype: int :param brk: the new address for C{brk}. :return: the value of the new C{brk}. :raises error: - \"Error in brk!\" if there is any error allocating the memory", "response": "def sys_brk(self, brk):\n        \"\"\"\n        Changes data segment size (moves the C{brk} to the new address)\n        :rtype: int\n        :param brk: the new address for C{brk}.\n        :return: the value of the new C{brk}.\n        :raises error:\n                    - \"Error in brk!\" if there is any error allocating the memory\n        \"\"\"\n        if brk != 0 and brk > self.elf_brk:\n            mem = self.current.memory\n            size = brk - self.brk\n            if brk > mem._ceil(self.brk):\n                perms = mem.perms(self.brk - 1)\n                addr = mem.mmap(mem._ceil(self.brk), size, perms)\n                if not mem._ceil(self.brk) == addr:\n                    logger.error(f\"Error in brk: ceil: {hex(mem._ceil(self.brk))} brk: {hex(brk)} self.brk: {hex(self.brk)} addr: {hex(addr)}\")\n                    return self.brk\n            self.brk += size\n        return self.brk"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sys_arch_prctl(self, code, addr):\n        ARCH_SET_GS = 0x1001\n        ARCH_SET_FS = 0x1002\n        ARCH_GET_FS = 0x1003\n        ARCH_GET_GS = 0x1004\n        if code not in {ARCH_SET_GS, ARCH_SET_FS, ARCH_GET_FS, ARCH_GET_GS}:\n            logger.debug(\"code not in expected options ARCH_GET/SET_FS/GS\")\n            return -errno.EINVAL\n        if code != ARCH_SET_FS:\n            raise NotImplementedError(\"Manticore supports only arch_prctl with code=ARCH_SET_FS (0x1002) for now\")\n        self.current.FS = 0x63\n        self.current.set_descriptor(self.current.FS, addr, 0x4000, 'rw')\n        return 0", "response": "Sets the architecture - specific thread state."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef sys_open(self, buf, flags, mode):\n        filename = self.current.read_string(buf)\n        try:\n            f = self._sys_open_get_file(filename, flags)\n            logger.debug(f\"Opening file {filename} for real fd {f.fileno()}\")\n        except IOError as e:\n            logger.warning(f\"Could not open file {filename}. Reason: {e!s}\")\n            return -e.errno if e.errno is not None else -errno.EINVAL\n\n        return self._open(f)", "response": "Open a file and return its handle."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef sys_openat(self, dirfd, buf, flags, mode):\n\n        filename = self.current.read_string(buf)\n        dirfd = ctypes.c_int32(dirfd).value\n\n        if os.path.isabs(filename) or dirfd == self.FCNTL_FDCWD:\n            return self.sys_open(buf, flags, mode)\n\n        try:\n            dir_entry = self._get_fd(dirfd)\n        except FdError as e:\n            logger.info(\"openat: Not valid file descriptor. Returning EBADF\")\n            return -e.err\n\n        if not isinstance(dir_entry, Directory):\n            logger.info(\"openat: Not directory descriptor. Returning ENOTDIR\")\n            return -errno.ENOTDIR\n\n        dir_path = dir_entry.name\n\n        filename = os.path.join(dir_path, filename)\n        try:\n            f = self._sys_open_get_file(filename, flags)\n            logger.debug(f\"Opening file {filename} for real fd {f.fileno()}\")\n        except IOError as e:\n            logger.info(f\"Could not open file {filename}. Reason: {e!s}\")\n            return -e.errno if e.errno is not None else -errno.EINVAL\n\n        return self._open(f)", "response": "Openat SystemCall - Similar to sys. open but uses the same arguments as sys. open but uses the same arguments as sys. open but uses the same arguments as sys. open but uses the same arguments as sys. open."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nrename filename oldnamep to newnamep.", "response": "def sys_rename(self, oldnamep, newnamep):\n        \"\"\"\n        Rename filename `oldnamep` to `newnamep`.\n\n        :param int oldnamep: pointer to oldname\n        :param int newnamep: pointer to newname\n        \"\"\"\n        oldname = self.current.read_string(oldnamep)\n        newname = self.current.read_string(newnamep)\n\n        ret = 0\n        try:\n            os.rename(oldname, newname)\n        except OSError as e:\n            ret = -e.errno\n\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsynchronizing a file s in - core state with that on disk.", "response": "def sys_fsync(self, fd):\n        \"\"\"\n        Synchronize a file's in-core state with that on disk.\n        \"\"\"\n\n        ret = 0\n        try:\n            self.files[fd].sync()\n        except IndexError:\n            ret = -errno.EBADF\n        except FdError:\n            ret = -errno.EINVAL\n\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef sys_dup(self, fd):\n\n        if not self._is_fd_open(fd):\n            logger.info(\"DUP: Passed fd is not open. Returning EBADF\")\n            return -errno.EBADF\n\n        newfd = self._dup(fd)\n        return newfd", "response": "Duplicate an open file descriptor."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nduplicates a file descriptor to a newfd.", "response": "def sys_dup2(self, fd, newfd):\n        \"\"\"\n        Duplicates an open fd to newfd. If newfd is open, it is first closed\n        :rtype: int\n        :param fd: the open file descriptor to duplicate.\n        :param newfd: the file descriptor to alias the file described by fd.\n        :return: newfd.\n        \"\"\"\n        try:\n            file = self._get_fd(fd)\n        except FdError as e:\n            logger.info(\"DUP2: Passed fd is not open. Returning EBADF\")\n            return -e.err\n\n        soft_max, hard_max = self._rlimits[self.RLIMIT_NOFILE]\n        if newfd >= soft_max:\n            logger.info(\"DUP2: newfd is above max descriptor table size\")\n            return -errno.EBADF\n\n        if self._is_fd_open(newfd):\n            self._close(newfd)\n\n        if newfd >= len(self.files):\n            self.files.extend([None] * (newfd + 1 - len(self.files)))\n\n        self.files[newfd] = self.files[fd]\n\n        logger.debug('sys_dup2(%d,%d) -> %d', fd, newfd, newfd)\n        return newfd"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nclose a file descriptor.", "response": "def sys_close(self, fd):\n        \"\"\"\n        Closes a file descriptor\n        :rtype: int\n        :param fd: the file descriptor to close.\n        :return: C{0} on success.\n        \"\"\"\n        if self._is_fd_open(fd):\n            self._close(fd)\n        else:\n            return -errno.EBADF\n        logger.debug(f'sys_close({fd})')\n        return 0"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sys_readlink(self, path, buf, bufsize):\n        if bufsize <= 0:\n            return -errno.EINVAL\n        filename = self.current.read_string(path)\n        if filename == '/proc/self/exe':\n            data = os.path.abspath(self.program)\n        else:\n            data = os.readlink(filename)[:bufsize]\n        self.current.write_bytes(buf, data)\n        return len(data)", "response": "This function reads the link at the specified path and writes it to the buffer."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwrap for sys_mmap2. Returns the pgoff of the first page.", "response": "def sys_mmap_pgoff(self, address, size, prot, flags, fd, offset):\n        \"\"\"Wrapper for mmap2\"\"\"\n        return self.sys_mmap2(address, size, prot, flags, fd, offset)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef sys_mmap(self, address, size, prot, flags, fd, offset):\n\n        if address == 0:\n            address = None\n\n        cpu = self.current\n        if flags & 0x10:\n            cpu.memory.munmap(address, size)\n\n        perms = perms_from_protflags(prot)\n\n        if flags & 0x20:\n            result = cpu.memory.mmap(address, size, perms)\n        elif fd == 0:\n            assert offset == 0\n            result = cpu.memory.mmap(address, size, perms)\n            data = self.files[fd].read(size)\n            cpu.write_bytes(result, data)\n        else:\n            # FIXME Check if file should be symbolic input and do as with fd0\n            result = cpu.memory.mmapFile(address, size, perms, self.files[fd].name, offset)\n\n        actually_mapped = f'0x{result:016x}'\n        if address is None or result != address:\n            address = address or 0\n            actually_mapped += f' [requested: 0x{address:016x}]'\n\n        if flags & 0x10 != 0 and result != address:\n            cpu.memory.munmap(result, size)\n            result = -1\n\n        return result", "response": "This function maps memory space to the virtual address space of the calling process."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef sys_mprotect(self, start, size, prot):\n        perms = perms_from_protflags(prot)\n        ret = self.current.memory.mprotect(start, size, perms)\n        return 0", "response": "Sets protection on a region of memory."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nworks just like C{sys_read} except that data is read into multiple buffers. :rtype: int :param fd: the file descriptor of the file to read. :param iov: the buffer where the the bytes to read are stored. :param count: amount of C{iov} buffers to read from the file. :return: the amount of bytes read in total.", "response": "def sys_readv(self, fd, iov, count):\n        \"\"\"\n        Works just like C{sys_read} except that data is read into multiple buffers.\n        :rtype: int\n\n        :param fd: the file descriptor of the file to read.\n        :param iov: the buffer where the the bytes to read are stored.\n        :param count: amount of C{iov} buffers to read from the file.\n        :return: the amount of bytes read in total.\n        \"\"\"\n        cpu = self.current\n        ptrsize = cpu.address_bit_size\n        sizeof_iovec = 2 * (ptrsize // 8)\n        total = 0\n        for i in range(0, count):\n            buf = cpu.read_int(iov + i * sizeof_iovec, ptrsize)\n            size = cpu.read_int(iov + i * sizeof_iovec + (sizeof_iovec // 2),\n                                ptrsize)\n\n            data = self.files[fd].read(size)\n            total += len(data)\n            cpu.write_bytes(buf, data)\n            self.syscall_trace.append((\"_read\", fd, data))\n        return total"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nwork just like C{sys_write} except that multiple buffers are written out. :rtype: int :param fd: the file descriptor of the file to write. :param iov: the buffer where the the bytes to write are taken. :param count: amount of C{iov} buffers to write into the file. :return: the amount of bytes written in total.", "response": "def sys_writev(self, fd, iov, count):\n        \"\"\"\n        Works just like C{sys_write} except that multiple buffers are written out.\n        :rtype: int\n\n        :param fd: the file descriptor of the file to write.\n        :param iov: the buffer where the the bytes to write are taken.\n        :param count: amount of C{iov} buffers to write into the file.\n        :return: the amount of bytes written in total.\n        \"\"\"\n        cpu = self.current\n        ptrsize = cpu.address_bit_size\n        sizeof_iovec = 2 * (ptrsize // 8)\n        total = 0\n        try:\n            write_fd = self._get_fd(fd)\n        except FdError as e:\n            logger.error(f\"writev: Not a valid file descriptor ({fd})\")\n            return -e.err\n\n        for i in range(0, count):\n            buf = cpu.read_int(iov + i * sizeof_iovec, ptrsize)\n            size = cpu.read_int(iov + i * sizeof_iovec + (sizeof_iovec // 2), ptrsize)\n\n            data = [Operators.CHR(cpu.read_int(buf + i, 8)) for i in range(size)]\n            data = self._transform_write_data(data)\n            write_fd.write(data)\n            self.syscall_trace.append((\"_write\", fd, data))\n            total += size\n        return total"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef sys_set_thread_area(self, user_info):\n        n = self.current.read_int(user_info, 32)\n        pointer = self.current.read_int(user_info + 4, 32)\n        m = self.current.read_int(user_info + 8, 32)\n        flags = self.current.read_int(user_info + 12, 32)\n        assert n == 0xffffffff\n        assert flags == 0x51  # TODO: fix\n        self.current.GS = 0x63\n        self.current.set_descriptor(self.current.GS, pointer, 0x4000, 'rw')\n        self.current.write_int(user_info, (0x63 - 3) // 8, 32)\n        return 0", "response": "Sets a thread local storage area."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef syscall(self):\n\n        index = self._syscall_abi.syscall_number()\n\n        try:\n            table = getattr(linux_syscalls, self.current.machine)\n            name = table.get(index, None)\n            implementation = getattr(self, name)\n        except (AttributeError, KeyError):\n            if name is not None:\n                raise SyscallNotImplemented(index, name)\n            else:\n                raise Exception(f\"Bad syscall index, {index}\")\n\n        return self._syscall_abi.invoke(implementation)", "response": "Returns the syscall that is currently running on this machine."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nyielding CPU. This will choose another process from the running list and change current running process.", "response": "def sched(self):\n        \"\"\" Yield CPU.\n            This will choose another process from the running list and change\n            current running process. May give the same cpu if only one running\n            process.\n        \"\"\"\n        if len(self.procs) > 1:\n            logger.debug(\"SCHED:\")\n            logger.debug(f\"\\tProcess: {self.procs!r}\")\n            logger.debug(f\"\\tRunning: {self.running!r}\")\n            logger.debug(f\"\\tRWait: {self.rwait!r}\")\n            logger.debug(f\"\\tTWait: {self.twait!r}\")\n            logger.debug(f\"\\tTimers: {self.timers!r}\")\n            logger.debug(f\"\\tCurrent clock: {self.clocks}\")\n            logger.debug(f\"\\tCurrent cpu: {self._current}\")\n\n        if len(self.running) == 0:\n            logger.debug(\"None running checking if there is some process waiting for a timeout\")\n            if all([x is None for x in self.timers]):\n                raise Deadlock()\n            self.clocks = min(x for x in self.timers if x is not None) + 1\n            self.check_timers()\n            assert len(self.running) != 0, \"DEADLOCK!\"\n            self._current = self.running[0]\n            return\n        next_index = (self.running.index(self._current) + 1) % len(self.running)\n        next_running_idx = self.running[next_index]\n        if len(self.procs) > 1:\n            logger.debug(f\"\\tTransfer control from process {self._current} to {next_running_idx}\")\n        self._current = next_running_idx"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef wait(self, readfds, writefds, timeout):\n        logger.debug(\"WAIT:\")\n        logger.debug(f\"\\tProcess {self._current} is going to wait for [ {readfds!r} {writefds!r} {timeout!r} ]\")\n        logger.debug(f\"\\tProcess: {self.procs!r}\")\n        logger.debug(f\"\\tRunning: {self.running!r}\")\n        logger.debug(f\"\\tRWait: {self.rwait!r}\")\n        logger.debug(f\"\\tTWait: {self.twait!r}\")\n        logger.debug(f\"\\tTimers: {self.timers!r}\")\n\n        for fd in readfds:\n            self.rwait[fd].add(self._current)\n        for fd in writefds:\n            self.twait[fd].add(self._current)\n        if timeout is not None:\n            self.timers[self._current] = self.clocks + timeout\n        procid = self._current\n        # self.sched()\n        next_index = (self.running.index(procid) + 1) % len(self.running)\n        self._current = self.running[next_index]\n        logger.debug(f\"\\tTransfer control from process {procid} to {self._current}\")\n        logger.debug(f\"\\tREMOVING {procid!r} from {self.running!r}. Current: {self._current!r}\")\n        self.running.remove(procid)\n        if self._current not in self.running:\n            logger.debug(\"\\tCurrent not running. Checking for timers...\")\n            self._current = None\n            self.check_timers()", "response": "Wait for a set of file descriptors or timeout."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef awake(self, procid):\n        logger.debug(f\"Remove procid:{procid} from waitlists and reestablish it in the running list\")\n        for wait_list in self.rwait:\n            if procid in wait_list:\n                wait_list.remove(procid)\n        for wait_list in self.twait:\n            if procid in wait_list:\n                wait_list.remove(procid)\n        self.timers[procid] = None\n        self.running.append(procid)\n        if self._current is None:\n            self._current = procid", "response": "Remove the procid from waitlists and reestablish it in the running list"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nsignal that a process is waiting to receive data on fd", "response": "def signal_receive(self, fd):\n        \"\"\" Awake one process waiting to receive data on fd \"\"\"\n        connections = self.connections\n        if connections(fd) and self.twait[connections(fd)]:\n            procid = random.sample(self.twait[connections(fd)], 1)[0]\n            self.awake(procid)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef signal_transmit(self, fd):\n        connection = self.connections(fd)\n        if connection is None or connection >= len(self.rwait):\n            return\n\n        procs = self.rwait[connection]\n        if procs:\n            procid = random.sample(procs, 1)[0]\n            self.awake(procid)", "response": "Signal that a process is waiting to transmit data on fd."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef check_timers(self):\n        if self._current is None:\n            # Advance the clocks. Go to future!!\n            advance = min([self.clocks] + [x for x in self.timers if x is not None]) + 1\n            logger.debug(f\"Advancing the clock from {self.clocks} to {advance}\")\n            self.clocks = advance\n        for procid in range(len(self.timers)):\n            if self.timers[procid] is not None:\n                if self.clocks > self.timers[procid]:\n                    self.procs[procid].PC += self.procs[procid].instruction.size\n                    self.awake(procid)", "response": "Awake process if timer has expired"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nexecuting one cpu instruction in the current thread.", "response": "def execute(self):\n        \"\"\"\n        Execute one cpu instruction in the current thread (only one supported).\n        :rtype: bool\n        :return: C{True}\n\n        :todo: This is where we could implement a simple schedule.\n        \"\"\"\n        try:\n            self.current.execute()\n            self.clocks += 1\n            if self.clocks % 10000 == 0:\n                self.check_timers()\n                self.sched()\n        except (Interruption, Syscall) as e:\n            try:\n                self.syscall()\n                if hasattr(e, 'on_handled'):\n                    e.on_handled()\n            except RestartSyscall:\n                pass\n\n        return True"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ndetermine information about a file based on its file descriptor (for Linux 64 bits). :rtype: int :param fd: the file descriptor of the file that is being inquired. :param buf: a buffer where data about the file will be stored. :return: C{0} on success, EBADF when called with bad fd :todo: Fix device number.", "response": "def sys_fstat64(self, fd, buf):\n        \"\"\"\n        Determines information about a file based on its file descriptor (for Linux 64 bits).\n        :rtype: int\n        :param fd: the file descriptor of the file that is being inquired.\n        :param buf: a buffer where data about the file will be stored.\n        :return: C{0} on success, EBADF when called with bad fd\n        :todo: Fix device number.\n        \"\"\"\n\n        try:\n            stat = self._get_fd(fd).stat()\n        except FdError as e:\n            logger.info(\"Calling fstat with invalid fd, returning EBADF\")\n            return -e.err\n\n        def add(width, val):\n            fformat = {2: 'H', 4: 'L', 8: 'Q'}[width]\n            return struct.pack('<' + fformat, val)\n\n        def to_timespec(ts):\n            return struct.pack('<LL', int(ts), int(ts % 1 * 1e9))\n\n        bufstat = add(8, stat.st_dev)        # unsigned long long      st_dev;\n        bufstat += add(8, stat.st_ino)        # unsigned long long   __st_ino;\n        bufstat += add(4, stat.st_mode)       # unsigned int    st_mode;\n        bufstat += add(4, stat.st_nlink)      # unsigned int    st_nlink;\n        bufstat += add(4, stat.st_uid)        # unsigned long   st_uid;\n        bufstat += add(4, stat.st_gid)        # unsigned long   st_gid;\n        bufstat += add(8, stat.st_rdev)       # unsigned long long st_rdev;\n        bufstat += add(8, 0)                  # unsigned long long __pad1;\n        bufstat += add(8, stat.st_size)       # long long       st_size;\n        bufstat += add(4, stat.st_blksize)    # int   st_blksize;\n        bufstat += add(4, 0)                  # int   __pad2;\n        bufstat += add(8, stat.st_blocks)     # unsigned long long st_blocks;\n        bufstat += to_timespec(stat.st_atime)  # unsigned long   st_atime;\n        bufstat += to_timespec(stat.st_mtime)  # unsigned long   st_mtime;\n        bufstat += to_timespec(stat.st_ctime)  # unsigned long   st_ctime;\n        bufstat += add(4, 0)                   # unsigned int __unused4;\n        bufstat += add(4, 0)                   # unsigned int __unused5;\n\n        self.current.write_bytes(buf, bufstat)\n        return 0"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncompute total load size of interpreter.", "response": "def _interp_total_size(interp):\n        \"\"\"\n        Compute total load size of interpreter.\n\n        :param ELFFile interp: interpreter ELF .so\n        :return: total load size of interpreter, not aligned\n        :rtype: int\n        \"\"\"\n        load_segs = [x for x in interp.iter_segments() if x.header.p_type == 'PT_LOAD']\n        last = load_segs[-1]\n        return last.header.p_vaddr + last.header.p_memsz"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _read_string(self, cpu, buf):\n        filename = \"\"\n        for i in range(0, 1024):\n            c = Operators.CHR(cpu.read_int(buf + i, 8))\n            if c == '\\x00':\n                break\n            filename += c\n        return filename", "response": "Reads a null terminated concrete buffer form memory\n           "}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nload a CGC - ELF program in memory and prepares the initial CPU state and stack state.", "response": "def load(self, filename):\n        \"\"\"\n        Loads a CGC-ELF program in memory and prepares the initial CPU state\n        and the stack.\n\n        :param filename: pathname of the file to be executed.\n        \"\"\"\n        CGC_MIN_PAGE_SIZE = 4096\n        CGC_MIN_ALIGN = CGC_MIN_PAGE_SIZE\n        TASK_SIZE = 0x80000000\n\n        def CGC_PAGESTART(_v):\n            return ((_v) & ~ (CGC_MIN_ALIGN - 1))\n\n        def CGC_PAGEOFFSET(_v):\n            return ((_v) & (CGC_MIN_ALIGN - 1))\n\n        def CGC_PAGEALIGN(_v):\n            return (((_v) + CGC_MIN_ALIGN - 1) & ~(CGC_MIN_ALIGN - 1))\n\n        def BAD_ADDR(x):\n            return ((x) >= TASK_SIZE)\n\n        # load elf See https://github.com/CyberdyneNYC/linux-source-3.13.2-cgc/blob/master/fs/binfmt_cgc.c\n        # read the ELF object file\n        cgc = CGCElf(filename)\n        logger.info(\"Loading %s as a %s elf\" % (filename, cgc.arch))\n        # make cpu and memory (Only 1 thread in Decree)\n        cpu = self._mk_proc()\n\n        bss = brk = 0\n        start_code = 0xffffffff\n        end_code = start_data = end_data = 0\n\n        for (vaddr, memsz, perms, name, offset, filesz) in cgc.maps():\n            if vaddr < start_code:\n                start_code = vaddr\n            if start_data < vaddr:\n                start_data = vaddr\n\n            if vaddr > TASK_SIZE or filesz > memsz or \\\n                    memsz > TASK_SIZE or TASK_SIZE - memsz < vaddr:\n                raise Exception(\"Set_brk can never work. avoid overflows\")\n\n            # CGCMAP--\n            addr = None\n            if filesz > 0:\n                hint = CGC_PAGESTART(vaddr)\n                size = CGC_PAGEALIGN(filesz + CGC_PAGEOFFSET(vaddr))\n                offset = CGC_PAGESTART(offset)\n                addr = cpu.memory.mmapFile(hint, size, perms, name, offset)\n                assert not BAD_ADDR(addr)\n\n                lo = CGC_PAGEALIGN(vaddr + filesz)\n                hi = CGC_PAGEALIGN(vaddr + memsz)\n            else:\n                # for 0 filesz, we have to include the first page as bss.\n                lo = CGC_PAGESTART(vaddr + filesz)\n                hi = CGC_PAGEALIGN(vaddr + memsz)\n\n            # map anon pages for the rest (no prefault)\n            if hi - lo > 0:\n                zaddr = cpu.memory.mmap(lo, hi - lo, perms)\n                assert not BAD_ADDR(zaddr)\n\n            lo = vaddr + filesz\n            hi = CGC_PAGEALIGN(vaddr + memsz)\n            if hi - lo > 0:\n                old_perms = cpu.memory.perms(lo)\n                cpu.memory.mprotect(lo, hi - lo, 'rw')\n                try:\n                    cpu.memory[lo:hi] = '\\x00' * (hi - lo)\n                except Exception as e:\n                    logger.debug(\"Exception zeroing main elf fractional pages: %s\" % str(e))\n                cpu.memory.mprotect(lo, hi, old_perms)\n\n            if addr is None:\n                addr = zaddr\n            assert addr is not None\n\n            k = vaddr + filesz\n            if k > bss:\n                bss = k\n            if 'x' in perms and end_code < k:\n                end_code = k\n            if end_data < k:\n                end_data = k\n\n            k = vaddr + memsz\n            if k > brk:\n                brk = k\n\n        bss = brk\n        stack_base = 0xbaaaaffc\n        stack_size = 0x800000\n        stack = cpu.memory.mmap(0xbaaab000 - stack_size, stack_size, 'rwx') + stack_size - 4\n        assert (stack_base) in cpu.memory and (stack_base - stack_size + 4) in cpu.memory\n\n        # Only one thread in Decree\n        status, thread = next(cgc.threads())\n        assert status == 'Running'\n\n        logger.info(\"Setting initial cpu state\")\n        # set initial CPU state\n        cpu.write_register('EAX', 0x0)\n        cpu.write_register('ECX', cpu.memory.mmap(CGC_PAGESTART(0x4347c000),\n                                                  CGC_PAGEALIGN(4096 + CGC_PAGEOFFSET(0x4347c000)),\n                                                  'rwx'))\n        cpu.write_register('EDX', 0x0)\n        cpu.write_register('EBX', 0x0)\n        cpu.write_register('ESP', stack)\n        cpu.write_register('EBP', 0x0)\n        cpu.write_register('ESI', 0x0)\n        cpu.write_register('EDI', 0x0)\n        cpu.write_register('EIP', thread['EIP'])\n        cpu.write_register('RFLAGS', 0x202)\n        cpu.write_register('CS', 0x0)\n        cpu.write_register('SS', 0x0)\n        cpu.write_register('DS', 0x0)\n        cpu.write_register('ES', 0x0)\n        cpu.write_register('FS', 0x0)\n        cpu.write_register('GS', 0x0)\n\n        cpu.memory.mmap(0x4347c000, 0x1000, 'r')\n        # cpu.memory[0x4347c000:0x4347d000] = 'A' 0x1000\n\n        logger.info(\"Entry point: %016x\", cpu.EIP)\n        logger.info(\"Stack start: %016x\", cpu.ESP)\n        logger.info(\"Brk: %016x\", brk)\n        logger.info(\"Mappings:\")\n        for m in str(cpu.memory).split('\\n'):\n            logger.info(\"  %s\", m)\n        return [cpu]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nallocate - allocate virtual memory The allocate system call creates a new allocation in the virtual address space of the calling process. The length argument specifies the length of the allocation in bytes which will be rounded up to the hardware page size. The kernel chooses the address at which to create the allocation; the address of the new allocation is returned in *addr as the result of the call. All newly allocated memory is readable and writeable. In addition, the is_X argument is a boolean that allows newly allocated memory to be marked as executable (non-zero) or non-executable (zero). The allocate function is invoked through system call number 5. :param cpu: current CPU :param length: the length of the allocation in bytes :param isX: boolean that allows newly allocated memory to be marked as executable :param addr: the address of the new allocation is returned in *addr :return: On success, allocate returns zero and a pointer to the allocated area is returned in *addr. Otherwise, an error code is returned and *addr is undefined. EINVAL length is zero. EINVAL length is too large. EFAULT addr points to an invalid address. ENOMEM No memory is available or the process' maximum number of allocations would have been exceeded.", "response": "def sys_allocate(self, cpu, length, isX, addr):\n        \"\"\" allocate - allocate virtual memory\n\n           The  allocate  system call creates a new allocation in the virtual address\n           space of the calling process.  The length argument specifies the length of\n           the allocation in bytes which will be rounded up to the hardware page size.\n\n           The kernel chooses the address at which to create the allocation; the\n           address of the new allocation is returned in *addr as the result of the call.\n\n           All newly allocated memory is readable and writeable. In addition, the\n           is_X argument is a boolean that allows newly allocated memory to be marked\n           as executable (non-zero) or non-executable (zero).\n\n           The allocate function is invoked through system call number 5.\n\n           :param cpu: current CPU\n           :param length: the length of the allocation in bytes\n           :param isX: boolean that allows newly allocated memory to be marked as executable\n           :param addr: the address of the new allocation is returned in *addr\n\n           :return: On success, allocate returns zero and a pointer to the allocated area\n                               is returned in *addr.  Otherwise, an error code is returned\n                               and *addr is undefined.\n                   EINVAL   length is zero.\n                   EINVAL   length is too large.\n                   EFAULT   addr points to an invalid address.\n                   ENOMEM   No memory is available or the process' maximum number of allocations\n                            would have been exceeded.\n        \"\"\"\n        # TODO: check 4 bytes from addr\n        if addr not in cpu.memory:\n            logger.info(\"ALLOCATE: addr points to invalid address. Returning EFAULT\")\n            return Decree.CGC_EFAULT\n\n        perms = ['rw ', 'rwx'][bool(isX)]\n        try:\n            result = cpu.memory.mmap(None, length, perms)\n        except Exception as e:\n            logger.info(\"ALLOCATE exception %s. Returning ENOMEM %r\", str(e), length)\n            return Decree.CGC_ENOMEM\n        cpu.write_int(addr, result, 32)\n        logger.info(\"ALLOCATE(%d, %s, 0x%08x) -> 0x%08x\" % (length, perms, addr, result))\n        self.syscall_trace.append((\"_allocate\", -1, length))\n        return 0"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef sys_random(self, cpu, buf, count, rnd_bytes):\n\n        ret = 0\n        if count != 0:\n            if count > Decree.CGC_SSIZE_MAX or count < 0:\n                ret = Decree.CGC_EINVAL\n            else:\n                # TODO check count bytes from buf\n                if buf not in cpu.memory or (buf + count) not in cpu.memory:\n                    logger.info(\"RANDOM: buf points to invalid address. Returning EFAULT\")\n                    return Decree.CGC_EFAULT\n\n                with open(\"/dev/urandom\", \"rb\") as f:\n                    data = f.read(count)\n\n                self.syscall_trace.append((\"_random\", -1, data))\n                cpu.write_bytes(buf, data)\n\n        # TODO check 4 bytes from rx_bytes\n        if rnd_bytes:\n            if rnd_bytes not in cpu.memory:\n                logger.info(\"RANDOM: Not valid rnd_bytes. Returning EFAULT\")\n                return Decree.CGC_EFAULT\n            cpu.write_int(rnd_bytes, len(data), 32)\n\n        logger.info(\"RANDOM(0x%08x, %d, 0x%08x) -> <%s>)\" % (buf, count, rnd_bytes, repr(data[:10])))\n        return ret", "response": "This function is used to fill a random buffer with random data."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sys_receive(self, cpu, fd, buf, count, rx_bytes):\n        data = ''\n        if count != 0:\n            if not self._is_open(fd):\n                logger.info(\"RECEIVE: Not valid file descriptor on receive. Returning EBADF\")\n                return Decree.CGC_EBADF\n\n            # TODO check count bytes from buf\n            if buf not in cpu.memory:  # or not  buf+count in cpu.memory:\n                logger.info(\"RECEIVE: buf points to invalid address. Returning EFAULT\")\n                return Decree.CGC_EFAULT\n\n            #import random\n            #count = random.randint(1,count)\n            if fd > 2 and self.files[fd].is_empty():\n                cpu.PC -= cpu.instruction.size\n                self.wait([fd], [], None)\n                raise RestartSyscall()\n\n            # get some potential delay\n            # if random.randint(5) == 0 and count > 1:\n            #    count = count/2\n\n            # Read the data and put it in memory\n            data = self.files[fd].receive(count)\n            self.syscall_trace.append((\"_receive\", fd, data))\n            cpu.write_bytes(buf, data)\n\n            self.signal_receive(fd)\n\n        # TODO check 4 bytes from rx_bytes\n        if rx_bytes:\n            if rx_bytes not in cpu.memory:\n                logger.info(\"RECEIVE: Not valid file descriptor on receive. Returning EFAULT\")\n                return Decree.CGC_EFAULT\n            cpu.write_int(rx_bytes, len(data), 32)\n\n        logger.info(\"RECEIVE(%d, 0x%08x, %d, 0x%08x) -> <%s> (size:%d)\" % (fd, buf, count, rx_bytes, repr(data)[:min(count, 10)], len(data)))\n        return 0", "response": "This function handles the receive system call. It reads up to count bytes from a file descriptor fd to the memory buffer and sets rx_bytes to zero."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sys_transmit(self, cpu, fd, buf, count, tx_bytes):\n        data = []\n        if count != 0:\n\n            if not self._is_open(fd):\n                logger.error(\"TRANSMIT: Not valid file descriptor. Returning EBADFD %d\", fd)\n                return Decree.CGC_EBADF\n\n            # TODO check count bytes from buf\n            if buf not in cpu.memory or (buf + count) not in cpu.memory:\n                logger.debug(\"TRANSMIT: buf points to invalid address. Rerurning EFAULT\")\n                return Decree.CGC_EFAULT\n\n            if fd > 2 and self.files[fd].is_full():\n                cpu.PC -= cpu.instruction.size\n                self.wait([], [fd], None)\n                raise RestartSyscall()\n\n            for i in range(0, count):\n                value = Operators.CHR(cpu.read_int(buf + i, 8))\n                if not isinstance(value, str):\n                    logger.debug(\"TRANSMIT: Writing symbolic values to file %d\", fd)\n                    #value = str(value)\n                data.append(value)\n            self.files[fd].transmit(data)\n\n            logger.info(\"TRANSMIT(%d, 0x%08x, %d, 0x%08x) -> <%.24r>\" % (fd, buf, count, tx_bytes, ''.join([str(x) for x in data])))\n            self.syscall_trace.append((\"_transmit\", fd, data))\n            self.signal_transmit(fd)\n\n        # TODO check 4 bytes from tx_bytes\n        if tx_bytes:\n            if tx_bytes not in cpu.memory:\n                logger.debug(\"TRANSMIT: Not valid tx_bytes pointer on transmit. Returning EFAULT\")\n                return Decree.CGC_EFAULT\n            cpu.write_int(tx_bytes, len(data), 32)\n\n        return 0", "response": "Transmit a number of bytes from a memory buffer to a file descriptor."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef sys_terminate(self, cpu, error_code):\n        procid = self.procs.index(cpu)\n        self.sched()\n        self.running.remove(procid)\n        # self.procs[procid] = None #let it there so we can report?\n        if issymbolic(error_code):\n            logger.info(\"TERMINATE PROC_%02d with symbolic exit code [%d,%d]\", procid, solver.minmax(self.constraints, error_code))\n        else:\n            logger.info(\"TERMINATE PROC_%02d %x\", procid, error_code)\n        if len(self.running) == 0:\n            raise TerminateState(f'Process exited correctly. Code: {error_code}')\n        return error_code", "response": "Terminate all threads in a process."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef sys_deallocate(self, cpu, addr, size):\n        logger.info(\"DEALLOCATE(0x%08x, %d)\" % (addr, size))\n\n        if addr & 0xfff != 0:\n            logger.info(\"DEALLOCATE: addr is not page aligned\")\n            return Decree.CGC_EINVAL\n        if size == 0:\n            logger.info(\"DEALLOCATE:length is zero\")\n            return Decree.CGC_EINVAL\n        # unlikely AND WRONG!!!\n        # if addr > Decree.CGC_SSIZE_MAX or addr+size > Decree.CGC_SSIZE_MAX:\n        #    logger.info(\"DEALLOCATE: part of the region being deallocated is outside the valid address range of the process\")\n        #    return Decree.CGC_EINVAL\n\n        cpu.memory.munmap(addr, size)\n        self.syscall_trace.append((\"_deallocate\", -1, size))\n        return 0", "response": "This function deallocates memory for the specified memory area and returns the number of pages that were deallocated."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef sched(self):\n        if len(self.procs) > 1:\n            logger.info(\"SCHED:\")\n            logger.info(\"\\tProcess: %r\", self.procs)\n            logger.info(\"\\tRunning: %r\", self.running)\n            logger.info(\"\\tRWait: %r\", self.rwait)\n            logger.info(\"\\tTWait: %r\", self.twait)\n            logger.info(\"\\tTimers: %r\", self.timers)\n            logger.info(\"\\tCurrent clock: %d\", self.clocks)\n            logger.info(\"\\tCurrent cpu: %d\", self._current)\n\n        if len(self.running) == 0:\n            logger.info(\"None running checking if there is some process waiting for a timeout\")\n            if all([x is None for x in self.timers]):\n                raise Deadlock()\n            self.clocks = min([x for x in self.timers if x is not None]) + 1\n            self.check_timers()\n            assert len(self.running) != 0, \"DEADLOCK!\"\n            self._current = self.running[0]\n            return\n        next_index = (self.running.index(self._current) + 1) % len(self.running)\n        next = self.running[next_index]\n        if len(self.procs) > 1:\n            logger.info(\"\\tTransfer control from process %d to %d\", self._current, next)\n        self._current = next", "response": "Yield CPU.\n            This will choose another process from the RUNNNIG list and change\n            current running process."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef wait(self, readfds, writefds, timeout):\n        logger.info(\"WAIT:\")\n        logger.info(\"\\tProcess %d is going to wait for [ %r %r %r ]\", self._current, readfds, writefds, timeout)\n        logger.info(\"\\tProcess: %r\", self.procs)\n        logger.info(\"\\tRunning: %r\", self.running)\n        logger.info(\"\\tRWait: %r\", self.rwait)\n        logger.info(\"\\tTWait: %r\", self.twait)\n        logger.info(\"\\tTimers: %r\", self.timers)\n\n        for fd in readfds:\n            self.rwait[fd].add(self._current)\n        for fd in writefds:\n            self.twait[fd].add(self._current)\n        if timeout is not None:\n            self.timers[self._current] = self.clocks + timeout\n        else:\n            self.timers[self._current] = None\n        procid = self._current\n        # self.sched()\n        next_index = (self.running.index(procid) + 1) % len(self.running)\n        self._current = self.running[next_index]\n        logger.info(\"\\tTransfer control from process %d to %d\", procid, self._current)\n        logger.info(\"\\tREMOVING %r from %r. Current: %r\", procid, self.running, self._current)\n        self.running.remove(procid)\n        if self._current not in self.running:\n            logger.info(\"\\tCurrent not running. Checking for timers...\")\n            self._current = None\n            if all([x is None for x in self.timers]):\n                raise Deadlock()\n            self.check_timers()", "response": "Wait for filedescriptors or timeout."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef signal_transmit(self, fd):\n        connections = self.connections\n        if connections(fd) and self.rwait[connections(fd)]:\n            procid = random.sample(self.rwait[connections(fd)], 1)[0]\n            self.awake(procid)", "response": "Signal that a process is waiting to transmit data on fd"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nexecute one cpu instruction in the current thread.", "response": "def execute(self):\n        \"\"\"\n        Execute one cpu instruction in the current thread (only one supported).\n        :rtype: bool\n        :return: C{True}\n\n        :todo: This is where we could implement a simple schedule.\n        \"\"\"\n        try:\n            self.current.execute()\n            self.clocks += 1\n            if self.clocks % 10000 == 0:\n                self.check_timers()\n                self.sched()\n        except Interruption as e:\n            if e.N != 0x80:\n                raise\n            try:\n                self.int80(self.current)\n            except RestartSyscall:\n                pass\n\n        return True"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef sys_transmit(self, cpu, fd, buf, count, tx_bytes):\n        if issymbolic(fd):\n            logger.info(\"Ask to write to a symbolic file descriptor!!\")\n            cpu.PC = cpu.PC - cpu.instruction.size\n            raise SymbolicSyscallArgument(cpu, 0)\n\n        if issymbolic(buf):\n            logger.info(\"Ask to write to a symbolic buffer\")\n            cpu.PC = cpu.PC - cpu.instruction.size\n            raise SymbolicSyscallArgument(cpu, 1)\n\n        if issymbolic(count):\n            logger.info(\"Ask to write a symbolic number of bytes \")\n            cpu.PC = cpu.PC - cpu.instruction.size\n            raise SymbolicSyscallArgument(cpu, 2)\n\n        if issymbolic(tx_bytes):\n            logger.info(\"Ask to return size to a symbolic address \")\n            cpu.PC = cpu.PC - cpu.instruction.size\n            raise SymbolicSyscallArgument(cpu, 3)\n\n        return super().sys_transmit(cpu, fd, buf, count, tx_bytes)", "response": "This method is a convenience method that handles the sys_transmit system call."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef fromdescriptor(cls, desc):\n        type_, uri = ('fs', None) if desc is None else desc.split(':', 1)\n        for subclass in cls.__subclasses__():\n            if subclass.store_type == type_:\n                return subclass(uri)\n        raise NotImplementedError(f\"Storage type '{type_}' not supported.\")", "response": "Create a new store instance based on the given descriptor."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef save_value(self, key, value):\n        with self.save_stream(key) as s:\n            s.write(value)", "response": "Save an arbitrary serializable value under key."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nload an arbitrary value identified by key.", "response": "def load_value(self, key, binary=False):\n        \"\"\"\n        Load an arbitrary value identified by `key`.\n\n        :param str key: The key that identifies the value\n        :return: The loaded value\n        \"\"\"\n        with self.load_stream(key, binary=binary) as s:\n            return s.read()"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef save_stream(self, key, binary=False):\n        s = io.BytesIO() if binary else io.StringIO()\n        yield s\n        self.save_value(key, s.getvalue())", "response": "A context manager that saves a value into a managed file - like object."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef load_stream(self, key, binary=False):\n        value = self.load_value(key, binary=binary)\n        yield io.BytesIO(value) if binary else io.StringIO(value)", "response": "Load a managed file - like object from which the calling code can read the previously - serialized data."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsaves a state to storage.", "response": "def save_state(self, state, key):\n        \"\"\"\n        Save a state to storage.\n\n        :param manticore.core.StateBase state:\n        :param str key:\n        :return:\n        \"\"\"\n        with self.save_stream(key, binary=True) as f:\n            self._serializer.serialize(state, f)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nloads a state from storage.", "response": "def load_state(self, key, delete=True):\n        \"\"\"\n        Load a state from storage.\n\n        :param key: key that identifies state\n        :rtype: manticore.core.StateBase\n        \"\"\"\n        with self.load_stream(key, binary=True) as f:\n            state = self._serializer.deserialize(f)\n            if delete:\n                self.rm(key)\n            return state"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nyields a file object representing key", "response": "def save_stream(self, key, binary=False):\n        \"\"\"\n        Yield a file object representing `key`\n\n        :param str key: The file to save to\n        :param bool binary: Whether we should treat it as binary\n        :return:\n        \"\"\"\n        mode = 'wb' if binary else 'w'\n        with open(os.path.join(self.uri, key), mode) as f:\n            yield f"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nloading a file from the cache store.", "response": "def load_stream(self, key, binary=False):\n        \"\"\"\n        :param str key: name of stream to load\n        :param bool binary: Whether we should treat it as binary\n        :return:\n        \"\"\"\n        with open(os.path.join(self.uri, key), 'rb' if binary else 'r') as f:\n            yield f"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef rm(self, key):\n        path = os.path.join(self.uri, key)\n        os.remove(path)", "response": "Removes the file identified by key."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn just the filenames that match glob_str inside the store directory.", "response": "def ls(self, glob_str):\n        \"\"\"\n        Return just the filenames that match `glob_str` inside the store directory.\n\n        :param str glob_str: A glob string, i.e. 'state_*'\n        :return: list of matched keys\n        \"\"\"\n        path = os.path.join(self.uri, glob_str)\n        return [os.path.split(s)[1] for s in glob.glob(path)]"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets a unique state id.", "response": "def _get_id(self):\n        \"\"\"\n        Get a unique state id.\n\n        :rtype: int\n        \"\"\"\n        id_ = self._last_id.value\n        self._last_id.value += 1\n        return id_"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef load_state(self, state_id, delete=True):\n        return self._store.load_state(f'{self._prefix}{state_id:08x}{self._suffix}', delete=delete)", "response": "Load a state from storage identified by state_id."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsave a state to storage.", "response": "def save_state(self, state, state_id=None):\n        \"\"\"\n        Save a state to storage, return identifier.\n\n        :param state: The state to save\n        :param int state_id: If not None force the state id potentially overwriting old states\n        :return: New state id\n        :rtype: int\n        \"\"\"\n        assert isinstance(state, StateBase)\n        if state_id is None:\n            state_id = self._get_id()\n        else:\n            self.rm_state(state_id)\n\n        self._store.save_state(state, f'{self._prefix}{state_id:08x}{self._suffix}')\n        return state_id"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate an indexed output stream i. e. test_00000001. name", "response": "def _named_stream(self, name, binary=False):\n        \"\"\"\n        Create an indexed output stream i.e. 'test_00000001.name'\n\n        :param name: Identifier for the stream\n        :return: A context-managed stream-like object\n        \"\"\"\n        with self._store.save_stream(self._named_key(name), binary=binary) as s:\n            yield s"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef t_INTN(t):\n    r\"int(?P<size>256|248|240|232|224|216|208|200|192|184|176|168|160|152|144|136|128|120|112|104|96|88|80|72|64|56|48|40|32|24|16|8)\"\n    size = int(t.lexer.lexmatch.group('size'))\n    t.value = ('int', size)\n    return t", "response": "r IntN | Int64 | Int32 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64 | Int64"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef t_UFIXEDMN(t):\n    r\"ufixed(?P<M>256|248|240|232|224|216|208|200|192|184|176|168|160|152|144|136|128|120|112|104|96|88|80|72|64|56|48|40|32|24|16|8)x(?P<N>80|79|78|77|76|75|74|73|72|71|70|69|68|67|66|65|64|63|62|61|60|59|58|57|56|55|54|53|52|51|50|49|48|47|46|45|44|43|42|41|40|39|38|37|36|35|34|33|32|31|30|29|28|27|26|25|24|23|22|21|20|19|18|17|16|15|14|13|12|11|10|9|8|7|6|5|4|3|2|1)\"\n    M = int(t.lexer.lexmatch.group('M'))\n    N = int(t.lexer.lexmatch.group('N'))\n    t.value = (\"ufixed\", M, N)\n    return t", "response": "t_UFIXEDMN - Matches fixed M and N."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef p_dynamic_fixed_type(p):\n    reps = int(p[3])\n    base_type = p[1]\n    p[0] = ('array', reps, base_type)", "response": "Dynamically set the dynamic type of the base element."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cmp_regs(cpu, should_print=False):\n    differing = False\n    gdb_regs = gdb.getCanonicalRegisters()\n    for name in sorted(gdb_regs):\n        vg = gdb_regs[name]\n        if name.endswith('psr'):\n            name = 'apsr'\n        v = cpu.read_register(name.upper())\n        if should_print:\n            logger.debug(f'{name} gdb:{vg:x} mcore:{v:x}')\n        if vg != v:\n            if should_print:\n                logger.warning('^^ unequal')\n            differing = True\n    if differing:\n        logger.debug(qemu.correspond(None))\n    return differing", "response": "Compare the canonical registers of a single object to the current one."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsynchronize the state of the current state of the current state of the Manticore and the GDB state if necessary.", "response": "def post_mcore(state, last_instruction):\n    \"\"\"\n    Handle syscalls (import memory) and bail if we diverge\n    \"\"\"\n    global in_helper\n\n    # Synchronize qemu state to manticore's after a system call\n    if last_instruction.mnemonic.lower() == 'svc':\n        # Synchronize all writes that have happened\n        writes = state.cpu.memory.pop_record_writes()\n        if writes:\n            logger.debug(\"Got %d writes\", len(writes))\n        for addr, val in writes:\n            gdb.setByte(addr, val[0])\n\n        # Write return val to gdb\n        gdb_r0 = gdb.getR('R0')\n        if gdb_r0 != state.cpu.R0:\n            logger.debug(f\"Writing 0x{state.cpu.R0:x} to R0 (overwriting 0x{gdb.getR('R0'):x})\")\n        for reg in state.cpu.canonical_registers:\n            if reg.endswith('PSR') or reg in ('R15', 'PC'):\n                continue\n            val = state.cpu.read_register(reg)\n            gdb.setR(reg, val)\n\n\n    # Ignore Linux kernel helpers\n    if (state.cpu.PC >> 16) == 0xffff:\n        in_helper = True\n        return\n\n    # If we executed a few instructions of a helper, we need to sync Manticore's\n    # state to GDB as soon as we stop executing a helper.\n    if in_helper:\n        for reg in state.cpu.canonical_registers:\n            if reg.endswith('PSR'):\n                continue\n            # Don't sync pc\n            if reg == 'R15':\n                continue\n            gdb.setR(reg, state.cpu.read_register(reg))\n        in_helper = False\n\n    if cmp_regs(state.cpu):\n        cmp_regs(state.cpu, should_print=True)\n        state.abandon()"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef sync_svc(state):\n    syscall = state.cpu.R7 # Grab idx from manticore since qemu could have exited\n    name = linux_syscalls.armv7[syscall]\n\n    logger.debug(f\"Syncing syscall: {name}\")\n\n    try:\n        # Make sure mmap returns the same address\n        if 'mmap' in name:\n            returned = gdb.getR('R0')\n            logger.debug(f\"Syncing mmap ({returned:x})\")\n            state.cpu.write_register('R0', returned)\n        if 'exit' in name:\n            return\n    except ValueError:\n        for reg in state.cpu.canonical_registers:\n            print(f'{reg}: {state.cpu.read_register(reg):x}')\n        raise", "response": "Synchronize some service calls in manticore."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsynchronize the stack and register state.", "response": "def initialize(state):\n    \"\"\"\n    Synchronize the stack and register state (manticore->qemu)\n    \"\"\"\n    logger.debug(f\"Copying {stack_top - state.cpu.SP} bytes in the stack..\")\n    stack_bottom = min(state.cpu.SP, gdb.getR('SP'))\n    for address in range(stack_bottom, stack_top):\n        b = state.cpu.read_int(address, 8)\n        gdb.setByte(address, chr(b))\n\n    logger.debug(\"Done\")\n\n    # Qemu fd's start at 5, ours at 3. Add two filler fds\n    mcore_stdout = state.platform.files[1]\n    state.platform.files.append(mcore_stdout)\n    state.platform.files.append(mcore_stdout)\n\n    # Sync gdb's regs\n    for gdb_reg in gdb.getCanonicalRegisters():\n        if gdb_reg.endswith('psr'):\n            mcore_reg = 'APSR'\n        else:\n            mcore_reg = gdb_reg.upper()\n        value = state.cpu.read_register(mcore_reg)\n        gdb.setR(gdb_reg, value)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nsimplify the expression to a Constant and returns the actual concrete value.", "response": "def to_constant(expression):\n    \"\"\"\n        Iff the expression can be simplified to a Constant get the actual concrete value.\n        This discards/ignore any taint\n    \"\"\"\n    value = simplify(expression)\n    if isinstance(value, Expression) and value.taint:\n        raise ValueError(\"Can not simplify tainted values to constant\")\n    if isinstance(value, Constant):\n        return value.value\n    elif isinstance(value, Array):\n        if expression.index_max:\n            ba = bytearray()\n            for i in range(expression.index_max):\n                value_i = simplify(value[i])\n                if not isinstance(value_i, Constant):\n                    break\n                ba.append(value_i.value)\n            else:\n                return bytes(ba)\n            return expression\n    return value"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _method(self, expression, *args):\n        assert expression.__class__.__mro__[-1] is object\n        for cls in expression.__class__.__mro__:\n            sort = cls.__name__\n            methodname = 'visit_%s' % sort\n            method = getattr(self, methodname, None)\n            if method is not None:\n                method(expression, *args)\n                return\n        return", "response": "Overload Visitor. _method because we want to iterate over the the\n            classes and call the method on them."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef visit_Operation(self, expression, *operands):\n        operation = self.operations.get(type(expression), None)\n        if operation is not None and \\\n                all(isinstance(o, Constant) for o in operands):\n            value = operation(*(x.value for x in operands))\n            if isinstance(expression, BitVec):\n                return BitVecConstant(expression.size, value, taint=expression.taint)\n            else:\n                isinstance(expression, Bool)\n                return BoolConstant(value, taint=expression.taint)\n        else:\n            if any(operands[i] is not expression.operands[i] for i in range(len(operands))):\n                expression = self._rebuild(expression, operands)\n        return expression", "response": "visit an operation expression and return the result"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrebuild the expression if any operands are changed", "response": "def visit_Operation(self, expression, *operands):\n        \"\"\" constant folding, if all operands of an expression are a Constant do the math \"\"\"\n        if all(isinstance(o, Constant) for o in operands):\n            expression = constant_folder(expression)\n        if self._changed(expression, operands):\n            expression = self._rebuild(expression, operands)\n        return expression"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef visit_BitVecConcat(self, expression, *operands):\n        op = expression.operands[0]\n\n        value = None\n        end = None\n        begining = None\n        for o in operands:\n            # If found a non BitVecExtract, do not apply\n            if not isinstance(o, BitVecExtract):\n                return None\n            # Set the value for the first item\n            if value is None:\n                value = o.value\n                begining = o.begining\n                end = o.end\n            else:\n                # If concat of extracts of different values do not apply\n                if value is not o.value:\n                    return None\n                # If concat of non contiguous extracs do not apply\n                if begining != o.end + 1:\n                    return None\n                # update begining variable\n                begining = o.begining\n\n        if value is not None:\n            if end + 1 == value.size and begining == 0:\n                return value\n            else:\n                return BitVecExtract(value, begining, end - begining + 1, taint=expression.taint)", "response": "Return the value of the first bit - vector element that is a concatenation of the operands."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nvisits the BitVecExtract expression and return the corresponding object.", "response": "def visit_BitVecExtract(self, expression, *operands):\n        \"\"\" extract(sizeof(a), 0)(a)  ==> a\n            extract(16, 0)( concat(a,b,c,d) ) => concat(c, d)\n            extract(m,M)(and/or/xor a b ) => and/or/xor((extract(m,M) a) (extract(m,M) a)\n        \"\"\"\n        op = expression.operands[0]\n        begining = expression.begining\n        end = expression.end\n        size = end - begining + 1\n\n        # extract(sizeof(a), 0)(a)  ==> a\n        if begining == 0 and end + 1 == op.size:\n            return op\n        elif isinstance(op, BitVecExtract):\n            return BitVecExtract(op.value, op.begining + begining, size, taint=expression.taint)\n        elif isinstance(op, BitVecConcat):\n            new_operands = []\n            bitcount = 0\n            for item in reversed(op.operands):\n                if begining >= item.size:\n                    begining -= item.size\n                else:\n                    if bitcount < expression.size:\n                        new_operands.append(item)\n                    bitcount += item.size\n            if begining != expression.begining:\n                return BitVecExtract(BitVecConcat(sum([x.size for x in new_operands]), *reversed(new_operands)),\n                                     begining, expression.size, taint=expression.taint)\n        if isinstance(op, (BitVecAnd, BitVecOr, BitVecXor)):\n            bitoperand_a, bitoperand_b = op.operands\n            return op.__class__(BitVecExtract(bitoperand_a, begining, expression.size), BitVecExtract(bitoperand_b, begining, expression.size), taint=expression.taint)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the result of the addition of the two entries.", "response": "def visit_BitVecAdd(self, expression, *operands):\n        \"\"\" a + 0  ==> a\n            0 + a  ==> a\n        \"\"\"\n        left = expression.operands[0]\n        right = expression.operands[1]\n        if isinstance(right, BitVecConstant):\n            if right.value == 0:\n                return left\n        if isinstance(left, BitVecConstant):\n            if left.value == 0:\n                return right"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef visit_BitVecSub(self, expression, *operands):\n        left = expression.operands[0]\n        right = expression.operands[1]\n        if isinstance(left, BitVecAdd):\n            if self._same_constant(left.operands[0], right):\n                return left.operands[1]\n            elif self._same_constant(left.operands[1], right):\n                return left.operands[0]", "response": "Visit the sub expression."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef visit_BitVecOr(self, expression, *operands):\n        left = expression.operands[0]\n        right = expression.operands[1]\n        if isinstance(right, BitVecConstant):\n            if right.value == 0:\n                return left\n            elif right.value == left.mask:\n                return right\n            elif isinstance(left, BitVecOr):\n                left_left = left.operands[0]\n                left_right = left.operands[1]\n                if isinstance(right, Constant):\n                    return BitVecOr(left_left, (left_right | right), taint=expression.taint)\n        elif isinstance(left, BitVecConstant):\n            return BitVecOr(right, left, taint=expression.taint)", "response": "Visit the Or expression."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef visit_BitVecAnd(self, expression, *operands):\n        left = expression.operands[0]\n        right = expression.operands[1]\n        if isinstance(right, BitVecConstant):\n            if right.value == 0:\n                return right\n            elif right.value == right.mask:\n                return left\n            elif isinstance(left, BitVecAnd):\n                left_left = left.operands[0]\n                left_right = left.operands[1]\n                if isinstance(right, Constant):\n                    return BitVecAnd(left_left, left_right & right, taint=expression.taint)\n            elif isinstance(left, BitVecOr):\n                left_left = left.operands[0]\n                left_right = left.operands[1]\n                return BitVecOr(right & left_left, right & left_right, taint=expression.taint)\n\n        elif isinstance(left, BitVecConstant):\n            return BitVecAnd(right, left, taint=expression.taint)", "response": "visit_BitVecAnd returns a BitVecAnd expression"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef visit_BitVecShiftLeft(self, expression, *operands):\n        left = expression.operands[0]\n        right = expression.operands[1]\n        if isinstance(right, BitVecConstant):\n            if right.value == 0:\n                return left\n            elif right.value >= right.size:\n                return left", "response": "a << 0 => a                       remove zero\n            a << ct => 0 if ct > sizeof(a)    remove big constant shift"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef visit_ArraySelect(self, expression, *operands):\n        arr, index = operands\n        if isinstance(arr, ArrayVariable):\n            return\n\n        if isinstance(index, BitVecConstant):\n            ival = index.value\n\n            # props are slow and using them in tight loops should be avoided, esp when they offer no additional validation\n            # arr._operands[1] = arr.index, arr._operands[0] = arr.array\n            while isinstance(arr, ArrayStore) and isinstance(arr._operands[1], BitVecConstant) and arr._operands[1]._value != ival:\n                arr = arr._operands[0]  # arr.array\n\n        if isinstance(index, BitVecConstant) and isinstance(arr, ArrayStore) and isinstance(arr.index, BitVecConstant) and arr.index.value == index.value:\n            return arr.value\n        else:\n            if arr is not expression.array:\n                return arr.select(index)", "response": "Visit the ArraySelect expression and return the value of the array entry."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalculating the size of a static type.", "response": "def _type_size(ty):\n        \"\"\" Calculate `static` type size \"\"\"\n        if ty[0] in ('int', 'uint', 'bytesM', 'function'):\n            return 32\n        elif ty[0] in ('tuple'):\n            result = 0\n            for ty_i in ty[1]:\n                result += ABI._type_size(ty_i)\n            return result\n        elif ty[0] in ('array'):\n            rep = ty[1]\n            result = 32  # offset link\n            return result\n        elif ty[0] in ('bytes', 'string'):\n            result = 32  # offset link\n            return result\n        raise ValueError"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nbuilds transaction data from function signature and arguments", "response": "def function_call(type_spec, *args):\n        \"\"\"\n        Build transaction data from function signature and arguments\n        \"\"\"\n        m = re.match(r\"(?P<name>[a-zA-Z_][a-zA-Z_0-9]*)(?P<type>\\(.*\\))\", type_spec)\n        if not m:\n            raise EthereumError(\"Function signature expected\")\n\n        ABI._check_and_warn_num_args(type_spec, *args)\n\n        result = ABI.function_selector(type_spec)  # Funcid\n        result += ABI.serialize(m.group('type'), *args)\n        return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nserializes value using type specification in ty.", "response": "def serialize(ty, *values, **kwargs):\n        \"\"\"\n        Serialize value using type specification in ty.\n        ABI.serialize('int256', 1000)\n        ABI.serialize('(int, int256)', 1000, 2000)\n        \"\"\"\n        try:\n            parsed_ty = abitypes.parse(ty)\n        except Exception as e:\n            # Catch and rebrand parsing errors\n            raise EthereumError(str(e))\n\n        if parsed_ty[0] != 'tuple':\n            if len(values) > 1:\n                raise ValueError('too many values passed for non-tuple')\n            values = values[0]\n            if isinstance(values, str):\n                values = values.encode()\n        else:\n            # implement type forgiveness for bytesM/string types\n            # allow python strs also to be used for Solidity bytesM/string types\n            values = tuple(val.encode() if isinstance(val, str) else val for val in values)\n\n        result, dyn_result = ABI._serialize(parsed_ty, values)\n        return result + dyn_result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef function_selector(method_name_and_signature):\n        s = sha3.keccak_256()\n        s.update(method_name_and_signature.encode())\n        return bytes(s.digest()[:4])", "response": "Makes a function hash id from a method name and signature"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _serialize_uint(value, size=32, padding=0):\n        if size <= 0 or size > 32:\n            raise ValueError\n\n        from .account import EVMAccount  # because of circular import\n        if not isinstance(value, (int, BitVec, EVMAccount)):\n            raise ValueError\n        if issymbolic(value):\n            # FIXME This temporary array variable should be obtained from a specific constraint store\n            bytes = ArrayVariable(index_bits=256, index_max=32, value_bits=8, name='temp{}'.format(uuid.uuid1()))\n            if value.size <= size * 8:\n                value = Operators.ZEXTEND(value, size * 8)\n            else:\n                # automatically truncate, e.g. if they passed a BitVec(256) for an `address` argument (160 bits)\n                value = Operators.EXTRACT(value, 0, size * 8)\n            bytes = ArrayProxy(bytes.write_BE(padding, value, size))\n        else:\n            value = int(value)\n            bytes = bytearray()\n            for _ in range(padding):\n                bytes.append(0)\n            for position in reversed(range(size)):\n                bytes.append(Operators.EXTRACT(value, position * 8, 8))\n        assert len(bytes) == size + padding\n        return bytes", "response": "Translates a python integer or a BitVec into a 32 byte string MSB first"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nserializes a signed python integer or a BitVec into a 32 byte string MSB first.", "response": "def _serialize_int(value, size=32, padding=0):\n        \"\"\"\n        Translates a signed python integral or a BitVec into a 32 byte string, MSB first\n        \"\"\"\n        if size <= 0 or size > 32:\n            raise ValueError\n        if not isinstance(value, (int, BitVec)):\n            raise ValueError\n        if issymbolic(value):\n            buf = ArrayVariable(index_bits=256, index_max=32, value_bits=8, name='temp{}'.format(uuid.uuid1()))\n            value = Operators.SEXTEND(value, value.size, size * 8)\n            buf = ArrayProxy(buf.write_BE(padding, value, size))\n        else:\n            value = int(value)\n            buf = bytearray()\n            for _ in range(padding):\n                buf.append(0)\n\n            for position in reversed(range(size)):\n                buf.append(Operators.EXTRACT(value, position * 8, 8))\n        return buf"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _deserialize_uint(data, nbytes=32, padding=0, offset=0):\n        assert isinstance(data, (bytearray, Array))\n        value = ABI._readBE(data, nbytes, padding=True, offset=offset)\n        value = Operators.ZEXTEND(value, (nbytes + padding) * 8)\n        return value", "response": "Deserialize an unsigned integer from the data buffer starting at offset."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading a `nbytes` bytes long big endian signed integer from `data` starting at `offset` :param data: sliceable buffer; symbolic buffer of Eth ABI encoded data :param nbytes: number of bytes to read starting from least significant byte :rtype: int or Expression", "response": "def _deserialize_int(data, nbytes=32, padding=0):\n        \"\"\"\n        Read a `nbytes` bytes long big endian signed integer from `data` starting at `offset`\n\n        :param data: sliceable buffer; symbolic buffer of Eth ABI encoded data\n        :param nbytes: number of bytes to read starting from least significant byte\n        :rtype: int or Expression\n        \"\"\"\n        assert isinstance(data, (bytearray, Array))\n        value = ABI._readBE(data, nbytes, padding=True)\n        value = Operators.SEXTEND(value, nbytes * 8, (nbytes + padding) * 8)\n        if not issymbolic(value):\n            # sign bit on\n            if value & (1 << (nbytes * 8 - 1)):\n                value = -(((~value) + 1) & ((1 << (nbytes * 8)) - 1))\n        return value"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nmaking sure an EVM instruction has all of its arguments concretized according to provided policies. Example decoration: @concretized_args(size='ONE', address='') def LOG(self, address, size, *topics): ... The above will make sure that the |size| parameter to LOG is Concretized when symbolic according to the 'ONE' policy and concretize |address| with the default policy. :param policies: A kwargs list of argument names and their respective policies. Provide None or '' as policy to use default. :return: A function decorator", "response": "def concretized_args(**policies):\n    \"\"\"\n    Make sure an EVM instruction has all of its arguments concretized according to\n    provided policies.\n\n    Example decoration:\n\n        @concretized_args(size='ONE', address='')\n        def LOG(self, address, size, *topics):\n            ...\n\n    The above will make sure that the |size| parameter to LOG is Concretized when symbolic\n    according to the 'ONE' policy and concretize |address| with the default policy.\n\n    :param policies: A kwargs list of argument names and their respective policies.\n                         Provide None or '' as policy to use default.\n    :return: A function decorator\n    \"\"\"\n    def concretizer(func):\n        @wraps(func)\n        def wrapper(*args, **kwargs):\n            spec = inspect.getfullargspec(func)\n            for arg, policy in policies.items():\n                assert arg in spec.args, \"Concretizer argument not found in wrapped function.\"\n                # index is 0-indexed, but ConcretizeArgument is 1-indexed. However, this is correct\n                # since implementation method is always a bound method (self is param 0)\n                index = spec.args.index(arg)\n                if not issymbolic(args[index]):\n                    continue\n                if not policy:\n                    policy = 'SAMPLED'\n\n                if policy == \"ACCOUNTS\":\n                    value = args[index]\n                    world = args[0].world\n                    #special handler for EVM only policy\n                    cond = world._constraint_to_accounts(value, ty='both', include_zero=True)\n                    world.constraints.add(cond)\n                    policy = 'ALL'\n                raise ConcretizeArgument(index, policy=policy)\n            return func(*args, **kwargs)\n        wrapper.__signature__ = inspect.signature(func)\n        return wrapper\n    return concretizer"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a dictionary representation of the current object.", "response": "def to_dict(self, mevm):\n        \"\"\"\n        Only meant to be used with concrete Transaction objects! (after calling .concretize())\n        \"\"\"\n        return dict(type=self.sort,\n                    from_address=self.caller,\n                    from_name=mevm.account_name(self.caller),\n                    to_address=self.address,\n                    to_name=mevm.account_name(self.address),\n                    value=self.value,\n                    gas=self.gas,\n                    data=binascii.hexlify(self.data).decode())"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef dump(self, stream, state, mevm, conc_tx=None):\n        from ..ethereum import ABI  # circular imports\n        from ..ethereum.manticore import flagged\n\n        is_something_symbolic = False\n\n        if conc_tx is None:\n            conc_tx = self.concretize(state)\n\n        # The result if any RETURN or REVERT\n        stream.write(\"Type: %s (%d)\\n\" % (self.sort, self.depth))\n\n        caller_solution = conc_tx.caller\n\n        caller_name = mevm.account_name(caller_solution)\n        stream.write(\"From: %s(0x%x) %s\\n\" % (caller_name, caller_solution, flagged(issymbolic(self.caller))))\n\n        address_solution = conc_tx.address\n        address_name = mevm.account_name(address_solution)\n\n        stream.write(\"To: %s(0x%x) %s\\n\" % (address_name, address_solution, flagged(issymbolic(self.address))))\n        stream.write(\"Value: %d %s\\n\" % (conc_tx.value, flagged(issymbolic(self.value))))\n        stream.write(\"Gas used: %d %s\\n\" % (conc_tx.gas, flagged(issymbolic(self.gas))))\n\n        tx_data = conc_tx.data\n\n        stream.write(\"Data: 0x{} {}\\n\".format(binascii.hexlify(tx_data).decode(), flagged(issymbolic(self.data))))\n\n        if self.return_data is not None:\n            return_data = conc_tx.return_data\n\n            stream.write(\"Return_data: 0x{} {}\\n\".format(binascii.hexlify(return_data).decode(), flagged(issymbolic(self.return_data))))\n\n        metadata = mevm.get_metadata(self.address)\n        if self.sort == 'CREATE':\n            if metadata is not None:\n\n                conc_args_data = conc_tx.data[len(metadata._init_bytecode):]\n                arguments = ABI.deserialize(metadata.get_constructor_arguments(), conc_args_data)\n\n                # TODO confirm: arguments should all be concrete?\n\n                is_argument_symbolic = any(map(issymbolic, arguments))  # is this redundant since arguments are all concrete?\n                stream.write('Function call:\\n')\n                stream.write(\"Constructor(\")\n                stream.write(','.join(map(repr, map(state.solve_one, arguments))))  # is this redundant since arguments are all concrete?\n                stream.write(') -> %s %s\\n' % (self.result, flagged(is_argument_symbolic)))\n\n        if self.sort == 'CALL':\n            if metadata is not None:\n                calldata = conc_tx.data\n                is_calldata_symbolic = issymbolic(self.data)\n\n                function_id = calldata[:4]  # hope there is enough data\n                signature = metadata.get_func_signature(function_id)\n                function_name = metadata.get_func_name(function_id)\n                if signature:\n                    _, arguments = ABI.deserialize(signature, calldata)\n                else:\n                    arguments = (calldata,)\n\n                return_data = None\n                if self.result == 'RETURN':\n                    ret_types = metadata.get_func_return_types(function_id)\n                    return_data = conc_tx.return_data\n                    return_values = ABI.deserialize(ret_types, return_data)  # function return\n\n                is_return_symbolic = issymbolic(self.return_data)\n\n                stream.write('\\n')\n                stream.write(\"Function call:\\n\")\n                stream.write(\"%s(\" % function_name)\n                stream.write(','.join(map(repr, arguments)))\n                stream.write(') -> %s %s\\n' % (self.result, flagged(is_calldata_symbolic)))\n\n                if return_data is not None:\n                    if len(return_values) == 1:\n                        return_values = return_values[0]\n\n                    stream.write('return: %r %s\\n' % (return_values, flagged(is_return_symbolic)))\n                is_something_symbolic = is_calldata_symbolic or is_return_symbolic\n\n        stream.write('\\n\\n')\n        return is_something_symbolic", "response": "Dump the current state of the current transaction into the stream."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _get_memfee(self, address, size=1):\n        if not issymbolic(size) and size == 0:\n            return 0\n\n        address = self.safe_add(address, size)\n        allocated = self.allocated\n        GMEMORY = 3\n        GQUADRATICMEMDENOM = 512  # 1 gas per 512 quadwords\n        old_size = Operators.ZEXTEND(Operators.UDIV(self.safe_add(allocated, 31), 32), 512)\n        new_size = Operators.ZEXTEND(Operators.UDIV(self.safe_add(address, 31), 32), 512)\n\n        old_totalfee = self.safe_mul(old_size, GMEMORY) + Operators.UDIV(self.safe_mul(old_size, old_size), GQUADRATICMEMDENOM)\n        new_totalfee = self.safe_mul(new_size, GMEMORY) + Operators.UDIV(self.safe_mul(new_size, new_size), GQUADRATICMEMDENOM)\n        memfee = new_totalfee - old_totalfee\n        flag = Operators.UGT(new_totalfee, old_totalfee)\n        return Operators.ITEBV(512, size == 0, 0, Operators.ITEBV(512, flag, memfee, 0))", "response": "This calculates the amount of extra gas needed for accessing the previously unused memory."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreads size byte from bytecode.", "response": "def read_code(self, address, size=1):\n        \"\"\"\n        Read size byte from bytecode.\n        If less than size bytes are available result will be pad with \\x00\n        \"\"\"\n        assert address < len(self.bytecode)\n        value = self.bytecode[address:address + size]\n        if len(value) < size:\n            value += '\\x00' * (size - len(value))  # pad with null (spec)\n        return value"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef instruction(self):\n        # FIXME check if pc points to invalid instruction\n        # if self.pc >= len(self.bytecode):\n        #    return InvalidOpcode('Code out of range')\n        # if self.pc in self.invalid:\n        #    raise InvalidOpcode('Opcode inside a PUSH immediate')\n        try:\n            _decoding_cache = getattr(self, '_decoding_cache')\n        except Exception:\n            _decoding_cache = self._decoding_cache = {}\n\n        pc = self.pc\n        if isinstance(pc, Constant):\n            pc = pc.value\n\n        if pc in _decoding_cache:\n            return _decoding_cache[pc]\n\n        def getcode():\n            bytecode = self.bytecode\n            for pc_i in range(pc, len(bytecode)):\n                yield simplify(bytecode[pc_i]).value\n            while True:\n                yield 0\n        instruction = EVMAsm.disassemble_one(getcode(), pc=pc, fork=DEFAULT_FORK)\n        _decoding_cache[pc] = instruction\n        return instruction", "response": "Return the instruction that corresponds to this instruction."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\npush into the stack.", "response": "def _push(self, value):\n        \"\"\"\n        Push into the stack\n\n              ITEM0\n              ITEM1\n              ITEM2\n        sp->  {empty}\n        \"\"\"\n        assert isinstance(value, int) or isinstance(value, BitVec) and value.size == 256\n        if len(self.stack) >= 1024:\n            raise StackOverflow()\n\n        if isinstance(value, int):\n            value = value & TT256M1\n\n        value = simplify(value)\n        if isinstance(value, Constant) and not value.taint:\n            value = value.value\n        self.stack.append(value)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading a value from the top of the stack without removing it", "response": "def _top(self, n=0):\n        \"\"\"Read a value from the top of the stack without removing it\"\"\"\n        if len(self.stack) - n < 0:\n            raise StackUnderflow()\n        return self.stack[n - 1]"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _checkpoint(self):\n        #Fixme[felipe] add a with self.disabled_events context mangr to Eventful\n        if self._checkpoint_data is None:\n            if not self._published_pre_instruction_events:\n                self._published_pre_instruction_events = True\n                self._publish('will_decode_instruction', self.pc)\n                self._publish('will_execute_instruction', self.pc, self.instruction)\n                self._publish('will_evm_execute_instruction', self.instruction, self._top_arguments())\n\n            pc = self.pc\n            instruction = self.instruction\n            old_gas = self.gas\n            allocated = self._allocated\n            #FIXME Not clear which exception should trigger first. OOG or insuficient stack\n            # this could raise an insuficient stack exception\n            arguments = self._pop_arguments()\n            fee = self._calculate_gas(*arguments)\n            self._checkpoint_data = (pc, old_gas, instruction, arguments, fee, allocated)\n        return self._checkpoint_data", "response": "Save and return a state checkpoint previous to current instruction"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _rollback(self):\n        last_pc, last_gas, last_instruction, last_arguments, fee, allocated = self._checkpoint_data\n        self._push_arguments(last_arguments)\n        self._gas = last_gas\n        self._pc = last_pc\n        self._allocated = allocated\n        self._checkpoint_data = None", "response": "Revert the stack gas pc and memory allocation so it looks like before executing the instruction"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _check_jmpdest(self):\n        should_check_jumpdest = self._check_jumpdest\n        if issymbolic(should_check_jumpdest):\n            should_check_jumpdest_solutions = solver.get_all_values(self.constraints, should_check_jumpdest)\n            if len(should_check_jumpdest_solutions) != 1:\n                raise EthereumError(\"Conditional not concretized at JMPDEST check\")\n            should_check_jumpdest = should_check_jumpdest_solutions[0]\n\n        if should_check_jumpdest:\n            self._check_jumpdest = False\n\n            pc = self.pc.value if isinstance(self.pc, Constant) else self.pc\n\n            if pc not in self._valid_jumpdests:\n                raise InvalidOpcode()", "response": "Checks that the current instruction is a JUMPDEST."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _store(self, offset, value, size=1):\n        self.memory.write_BE(offset, value, size)\n        for i in range(size):\n            self._publish('did_evm_write_memory', offset + i, Operators.EXTRACT(value, (size - i - 1) * 8, 8))", "response": "Stores value in memory as a big endian"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef SDIV(self, a, b):\n        s0, s1 = to_signed(a), to_signed(b)\n        try:\n            result = (Operators.ABS(s0) // Operators.ABS(s1) * Operators.ITEBV(256, (s0 < 0) != (s1 < 0), -1, 1))\n        except ZeroDivisionError:\n            result = 0\n        result = Operators.ITEBV(256, b == 0, 0, result)\n        if not issymbolic(result):\n            result = to_signed(result)\n        return result", "response": "Signed integer division operation."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef SMOD(self, a, b):\n        s0, s1 = to_signed(a), to_signed(b)\n        sign = Operators.ITEBV(256, s0 < 0, -1, 1)\n        try:\n            result = (Operators.ABS(s0) % Operators.ABS(s1)) * sign\n        except ZeroDivisionError:\n            result = 0\n\n        return Operators.ITEBV(256, s1 == 0, 0, result)", "response": "Signed modulo remainder operation"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncalculating extra gas fee for a given base and exponent", "response": "def EXP_gas(self, base, exponent):\n        \"\"\"Calculate extra gas fee\"\"\"\n        EXP_SUPPLEMENTAL_GAS = 10   # cost of EXP exponent per byte\n\n        def nbytes(e):\n            result = 0\n            for i in range(32):\n                result = Operators.ITEBV(512, Operators.EXTRACT(e, i * 8, 8) != 0, i + 1, result)\n            return result\n        return EXP_SUPPLEMENTAL_GAS * nbytes(exponent)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef SIGNEXTEND(self, size, value):\n        # FIXME maybe use Operators.SEXTEND\n        testbit = Operators.ITEBV(256, size <= 31, size * 8 + 7, 257)\n        result1 = (value | (TT256 - (1 << testbit)))\n        result2 = (value & ((1 << testbit) - 1))\n        result = Operators.ITEBV(256, (value & (1 << testbit)) != 0, result1, result2)\n        return Operators.ITEBV(256, size <= 31, result, value)", "response": "Extend length of two s complement signed integer"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef LT(self, a, b):\n        return Operators.ITEBV(256, Operators.ULT(a, b), 1, 0)", "response": "get a less - than expression"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef GT(self, a, b):\n        return Operators.ITEBV(256, Operators.UGT(a, b), 1, 0)", "response": "get the greater - than comparison"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef SGT(self, a, b):\n        # http://gavwood.com/paper.pdf\n        s0, s1 = to_signed(a), to_signed(b)\n        return Operators.ITEBV(256, s0 > s1, 1, 0)", "response": "Signed greater - than comparison"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nretrieving single byte from word", "response": "def BYTE(self, offset, value):\n        \"\"\"Retrieve single byte from word\"\"\"\n        offset = Operators.ITEBV(256, offset < 32, (31 - offset) * 8, 256)\n        return Operators.ZEXTEND(Operators.EXTRACT(value, offset, 8), 256)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncompute the Keccak - 256 hash of the data in the memory starting at start and size.", "response": "def SHA3(self, start, size):\n        \"\"\"Compute Keccak-256 hash\"\"\"\n        # read memory from start to end\n        # http://gavwood.com/paper.pdf\n        data = self.try_simplify_to_constant(self.read_buffer(start, size))\n\n        if issymbolic(data):\n            known_sha3 = {}\n            # Broadcast the signal\n            self._publish('on_symbolic_sha3', data, known_sha3)  # This updates the local copy of sha3 with the pairs we need to explore\n\n            value = 0  # never used\n            known_hashes_cond = False\n            for key, hsh in known_sha3.items():\n                assert not issymbolic(key), \"Saved sha3 data,hash pairs should be concrete\"\n                cond = key == data\n                known_hashes_cond = Operators.OR(cond, known_hashes_cond)\n                value = Operators.ITEBV(256, cond, hsh, value)\n            return value\n\n        value = sha3.keccak_256(data).hexdigest()\n        value = int(value, 16)\n        self._publish('on_concrete_sha3', data, value)\n        logger.info(\"Found a concrete SHA3 example %r -> %x\", data, value)\n        return value"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngetting input data of current environment", "response": "def CALLDATALOAD(self, offset):\n        \"\"\"Get input data of current environment\"\"\"\n\n        if issymbolic(offset):\n            if solver.can_be_true(self._constraints, offset == self._used_calldata_size):\n                self.constraints.add(offset == self._used_calldata_size)\n            raise ConcretizeArgument(1, policy='SAMPLED')\n\n        self._use_calldata(offset, 32)\n\n        data_length = len(self.data)\n\n        bytes = []\n        for i in range(32):\n            try:\n                c = Operators.ITEBV(8, offset + i < data_length, self.data[offset + i], 0)\n            except IndexError:\n                # offset + i is concrete and outside data\n                c = 0\n\n            bytes.append(c)\n        return Operators.CONCAT(256, *bytes)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncopying input data in current environment to memory", "response": "def CALLDATACOPY(self, mem_offset, data_offset, size):\n        \"\"\"Copy input data in current environment to memory\"\"\"\n\n        if issymbolic(size):\n            if solver.can_be_true(self._constraints, size <= len(self.data) + 32):\n                self.constraints.add(size <= len(self.data) + 32)\n            raise ConcretizeArgument(3, policy='SAMPLED')\n\n        if issymbolic(data_offset):\n            if solver.can_be_true(self._constraints, data_offset == self._used_calldata_size):\n                self.constraints.add(data_offset == self._used_calldata_size)\n            raise ConcretizeArgument(2, policy='SAMPLED')\n\n        #account for calldata usage\n        self._use_calldata(data_offset, size)\n        self._allocate(mem_offset, size)\n        for i in range(size):\n            try:\n                c = Operators.ITEBV(8, data_offset + i < len(self.data), Operators.ORD(self.data[data_offset + i]), 0)\n            except IndexError:\n                # data_offset + i is concrete and outside data\n                c = 0\n            self._store(mem_offset + i, c)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef CODECOPY(self, mem_offset, code_offset, size):\n\n        self._allocate(mem_offset, size)\n        GCOPY = 3             # cost to copy one 32 byte word\n        copyfee = self.safe_mul(GCOPY, Operators.UDIV(self.safe_add(size, 31), 32))\n        self._consume(copyfee)\n\n        if issymbolic(size):\n            max_size = solver.max(self.constraints, size)\n        else:\n            max_size = size\n\n        for i in range(max_size):\n            if issymbolic(i < size):\n                default = Operators.ITEBV(8, i < size, 0, self._load(mem_offset + i, 1))  # Fixme. unnecessary memory read\n            else:\n                if i < size:\n                    default = 0\n                else:\n                    default = self._load(mem_offset + i, 1)\n\n            if issymbolic(code_offset):\n                value = Operators.ITEBV(8, code_offset + i >= len(self.bytecode), default, self.bytecode[code_offset + i])\n            else:\n                if code_offset + i >= len(self.bytecode):\n                    value = default\n                else:\n                    value = self.bytecode[code_offset + i]\n            self._store(mem_offset + i, value)\n        self._publish('did_evm_read_code', code_offset, size)", "response": "Copy code running in current environment to memory"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncopying an account s code to memory", "response": "def EXTCODECOPY(self, account, address, offset, size):\n        \"\"\"Copy an account's code to memory\"\"\"\n        extbytecode = self.world.get_code(account)\n        self._allocate(address + size)\n\n        for i in range(size):\n            if offset + i < len(extbytecode):\n                self._store(address + i, extbytecode[offset + i])\n            else:\n                self._store(address + i, 0)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nload word from memory.", "response": "def MLOAD(self, address):\n        \"\"\"Load word from memory\"\"\"\n        self._allocate(address, 32)\n        value = self._load(address, 32)\n        return value"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsaving word to memory", "response": "def MSTORE(self, address, value):\n        \"\"\"Save word to memory\"\"\"\n        if istainted(self.pc):\n            for taint in get_taints(self.pc):\n                value = taint_with(value, taint)\n        self._allocate(address, 32)\n        self._store(address, value, 32)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsave byte to memory.", "response": "def MSTORE8(self, address, value):\n        \"\"\"Save byte to memory\"\"\"\n        if istainted(self.pc):\n            for taint in get_taints(self.pc):\n                value = taint_with(value, taint)\n        self._allocate(address, 1)\n        self._store(address, Operators.EXTRACT(value, 0, 8), 1)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nload word from storage", "response": "def SLOAD(self, offset):\n        \"\"\"Load word from storage\"\"\"\n        storage_address = self.address\n        self._publish('will_evm_read_storage', storage_address, offset)\n        value = self.world.get_storage_data(storage_address, offset)\n        self._publish('did_evm_read_storage', storage_address, offset, value)\n        return value"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsaving word to storage", "response": "def SSTORE(self, offset, value):\n        \"\"\"Save word to storage\"\"\"\n        storage_address = self.address\n        self._publish('will_evm_write_storage', storage_address, offset, value)\n        #refund = Operators.ITEBV(256,\n        #                         previous_value != 0,\n        #                         Operators.ITEBV(256, value != 0, 0, GSTORAGEREFUND),\n        #                         0)\n\n        if istainted(self.pc):\n            for taint in get_taints(self.pc):\n                value = taint_with(value, taint)\n        self.world.set_storage_data(storage_address, offset, value)\n        self._publish('did_evm_write_storage', storage_address, offset, value)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nexchange 1st and 2nd stack items", "response": "def SWAP(self, *operands):\n        \"\"\"Exchange 1st and 2nd stack items\"\"\"\n        a = operands[0]\n        b = operands[-1]\n        return (b,) + operands[1:-1] + (a,)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef CREATE(self, value, offset, size):\n        address = self.world.create_account(address=EVMWorld.calculate_new_address(sender=self.address, nonce=self.world.get_nonce(self.address)))\n        self.world.start_transaction('CREATE',\n                                     address,\n                                     data=self.read_buffer(offset, size),\n                                     caller=self.address,\n                                     value=value,\n                                     gas=self.gas)\n\n        raise StartTx()", "response": "Create a new account with associated code"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a new account with associated code", "response": "def CREATE(self, value, offset, size):\n        \"\"\"Create a new account with associated code\"\"\"\n        tx = self.world.last_transaction  # At this point last and current tx are the same.\n        address = tx.address\n        if tx.result == 'RETURN':\n            self.world.set_code(tx.address, tx.return_data)\n        else:\n            self.world.delete_account(address)\n            address = 0\n        return address"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef CALLCODE(self, gas, _ignored_, value, in_offset, in_size, out_offset, out_size):\n        self.world.start_transaction('CALLCODE',\n                                     address=self.address,\n                                     data=self.read_buffer(in_offset, in_size),\n                                     caller=self.address,\n                                     value=value,\n                                     gas=gas)\n        raise StartTx()", "response": "Message - call into this account with alternative account s code"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef RETURN(self, offset, size):\n        data = self.read_buffer(offset, size)\n        raise EndTx('RETURN', data)", "response": "Halt execution returning output data"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef SELFDESTRUCT(self, recipient):\n        #This may create a user account\n        recipient = Operators.EXTRACT(recipient, 0, 160)\n        address = self.address\n        #FIXME for on the known addresses\n        if issymbolic(recipient):\n            logger.info(\"Symbolic recipient on self destruct\")\n            recipient = solver.get_value(self.constraints, recipient)\n\n        if recipient not in self.world:\n            self.world.create_account(address=recipient)\n\n        self.world.send_funds(address, recipient, self.world.get_balance(address))\n        self.world.delete_account(address)\n\n        raise EndTx('SELFDESTRUCT')", "response": "Halt execution and register account for later deletion"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef human_transactions(self):\n        txs = []\n        for tx in self.transactions:\n            if tx.depth == 0:\n                txs.append(tx)\n        return tuple(txs)", "response": "Returns a tuple of all human transactions."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef current_vm(self):\n        try:\n            _, _, _, _, vm = self._callstack[-1]\n            return vm\n        except IndexError:\n            return None", "response": "get the current vm"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the current transaction ID.", "response": "def current_transaction(self):\n        \"\"\"current tx\"\"\"\n        try:\n            tx, _, _, _, _ = self._callstack[-1]\n            if tx.result is not None:\n                #That tx finished. No current tx.\n                return None\n            return tx\n        except IndexError:\n            return None"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn the current human transaction ID.", "response": "def current_human_transaction(self):\n        \"\"\"Current ongoing human transaction\"\"\"\n        try:\n            tx, _, _, _, _ = self._callstack[0]\n            if tx.result is not None:\n                #That tx finished. No current tx.\n                return None\n            assert tx.depth == 0\n            return tx\n        except IndexError:\n            return None"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_storage_data(self, storage_address, offset):\n        value = self._world_state[storage_address]['storage'].get(offset, 0)\n        return simplify(value)", "response": "Read a value from a storage slot on the specified account"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwrite a value to a storage slot in the specified account", "response": "def set_storage_data(self, storage_address, offset, value):\n        \"\"\"\n        Writes a value to a storage slot in specified account\n\n        :param storage_address: an account address\n        :param offset: the storage slot to use.\n        :type offset: int or BitVec\n        :param value: the value to write\n        :type value: int or BitVec\n        \"\"\"\n        self._world_state[storage_address]['storage'][offset] = value"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get_storage_items(self, address):\n        storage = self._world_state[address]['storage']\n        items = []\n        array = storage.array\n        while not isinstance(array, ArrayVariable):\n            items.append((array.index, array.value))\n            array = array.array\n        return items", "response": "Gets all items in an account storage."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef has_storage(self, address):\n        storage = self._world_state[address]['storage']\n        array = storage.array\n        while not isinstance(array, ArrayVariable):\n            if isinstance(array, ArrayStore):\n                return True\n            array = array.array\n        return False", "response": "Returns True if something has been written to the storage."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncalculate a block s hash based on the block number.", "response": "def block_hash(self, block_number=None, force_recent=True):\n        \"\"\"\n        Calculates a block's hash\n        :param block_number: the block number for which to calculate the hash, defaulting to the most recent block\n        :param force_recent: if True (the default) return zero for any block that is in the future or older than 256 blocks\n        :return: the block hash\n        \"\"\"\n        if block_number is None:\n            block_number = self.block_number() - 1\n\n        # We are not maintaining an actual -block-chain- so we just generate\n        # some hashes for each virtual block\n        value = sha3.keccak_256((repr(block_number) + 'NONCE').encode()).hexdigest()\n        value = int(value, 16)\n\n        if force_recent:\n            # 0 is left on the stack if the looked for block number is greater or equal\n            # than the current block number or more than 256 blocks behind the current\n            # block. (Current block hash is unknown from inside the tx)\n            bnmax = Operators.ITEBV(256, self.block_number() > 256, 256, self.block_number())\n            value = Operators.ITEBV(256, Operators.OR(block_number >= self.block_number(), block_number < bnmax), 0, value)\n\n        return value"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a fresh 160bit address", "response": "def new_address(self, sender=None, nonce=None):\n        \"\"\"Create a fresh 160bit address\"\"\"\n        if sender is not None and nonce is None:\n            nonce = self.get_nonce(sender)\n\n        new_address = self.calculate_new_address(sender, nonce)\n        if sender is None and new_address in self:\n            return self.new_address(sender, nonce)\n        return new_address"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a new Wei - related account.", "response": "def create_account(self, address=None, balance=0, code=None, storage=None, nonce=None):\n        \"\"\"\n        Low level account creation. No transaction is done.\n        :param address: the address of the account, if known. If omitted, a new address will be generated as closely to the Yellow Paper as possible.\n        :param balance: the initial balance of the account in Wei\n        :param code: the runtime code of the account, if a contract\n        :param storage: storage array\n        :param nonce: the nonce for the account; contracts should have a nonce greater than or equal to 1\n        \"\"\"\n        if code is None:\n            code = bytes()\n        else:\n            if not isinstance(code, (bytes, Array)):\n                raise EthereumError('Wrong code type')\n\n        # nonce default to initial nonce\n        if nonce is None:\n            # As per EIP 161, contract accounts are initialized with a nonce of 1\n            nonce = 1 if code else 0\n\n        if address is None:\n            address = self.new_address()\n\n        if not isinstance(address, int):\n            raise EthereumError('You must provide an address')\n\n        if address in self.accounts:\n            # FIXME account may have been created via selfdestruct destination\n            # or CALL and may contain some ether already, though if it was a\n            # selfdestructed address, it can not be reused\n            raise EthereumError('The account already exists')\n\n        if storage is None:\n            # Uninitialized values in a storage are 0 by spec\n            storage = self.constraints.new_array(index_bits=256, value_bits=256, name=f'STORAGE_{address:x}', avoid_collisions=True, default=0)\n        else:\n            if isinstance(storage, ArrayProxy):\n                if storage.index_bits != 256 or storage.value_bits != 256:\n                    raise TypeError(\"An ArrayProxy 256bits -> 256bits is needed\")\n            else:\n                if any((k < 0 or k >= 1 << 256 for k, v in storage.items())):\n                    raise TypeError(\"Need a dict like object that maps 256 bits keys to 256 bits values\")\n            # Hopefully here we have a mapping from 256b to 256b\n\n        self._world_state[address] = {}\n        self._world_state[address]['nonce'] = nonce\n        self._world_state[address]['balance'] = balance\n        self._world_state[address]['storage'] = storage\n        self._world_state[address]['code'] = code\n\n        # adds hash of new address\n        data = binascii.unhexlify('{:064x}{:064x}'.format(address, 0))\n        value = sha3.keccak_256(data).hexdigest()\n        value = int(value, 16)\n        self._publish('on_concrete_sha3', data, value)\n\n        return address"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a contract account.", "response": "def create_contract(self, price=0, address=None, caller=None, balance=0, init=None, gas=None):\n        \"\"\"\n        Create a contract account. Sends a transaction to initialize the contract\n\n        :param address: the address of the new account, if known. If omitted, a new address will be generated as closely to the Yellow Paper as possible.\n        :param balance: the initial balance of the account in Wei\n        :param init: the initialization code of the contract\n\n        The way that the Solidity compiler expects the constructor arguments to\n        be passed is by appending the arguments to the byte code produced by the\n        Solidity compiler. The arguments are formatted as defined in the Ethereum\n        ABI2. The arguments are then copied from the init byte array to the EVM\n        memory through the CODECOPY opcode with appropriate values on the stack.\n        This is done when the byte code in the init byte array is actually run\n        on the network.\n        \"\"\"\n        expected_address = self.create_account(self.new_address(sender=caller))\n        if address is None:\n            address = expected_address\n        elif caller is not None and address != expected_address:\n            raise EthereumError(f\"Error: contract created from address {hex(caller)} with nonce {self.get_nonce(caller)} was expected to be at address {hex(expected_address)}, but create_contract was called with address={hex(address)}\")\n        self.start_transaction('CREATE', address, price, init, caller, balance, gas=gas)\n        self._process_pending_transaction()\n        return address"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ninitiates a new transaction for this account.", "response": "def start_transaction(self, sort, address, price=None, data=None, caller=None, value=0, gas=2300):\n        \"\"\"\n        Initiate a transaction\n        :param sort: the type of transaction. CREATE or CALL or DELEGATECALL\n        :param address: the address of the account which owns the code that is executing.\n        :param price: the price of gas in the transaction that originated this execution.\n        :param data: the byte array that is the input data to this execution\n        :param caller: the address of the account which caused the code to be executing. A 160-bit code used for identifying Accounts\n        :param value: the value, in Wei, passed to this account as part of the same procedure as execution. One Ether is defined as being 10**18 Wei.\n        :param bytecode: the byte array that is the machine code to be executed.\n        :param gas: gas budget for this transaction.\n        \"\"\"\n        assert self._pending_transaction is None, \"Already started tx\"\n        self._pending_transaction = PendingTransaction(sort, address, price, data, caller, value, gas)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _write_APSR(self, apsr):\n        V = Operators.EXTRACT(apsr, 28, 1)\n        C = Operators.EXTRACT(apsr, 29, 1)\n        Z = Operators.EXTRACT(apsr, 30, 1)\n        N = Operators.EXTRACT(apsr, 31, 1)\n\n        self.write('APSR_V', V)\n        self.write('APSR_C', C)\n        self.write('APSR_Z', Z)\n        self.write('APSR_N', N)", "response": "Auxiliary function - Writes flags from a full APSR"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _swap_mode(self):\n        assert self.mode in (cs.CS_MODE_ARM, cs.CS_MODE_THUMB)\n        if self.mode == cs.CS_MODE_ARM:\n            self.mode = cs.CS_MODE_THUMB\n        else:\n            self.mode = cs.CS_MODE_ARM", "response": "Toggle between ARM and Thumb mode"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsets the flags of the most recent APSR entries in the registry file.", "response": "def set_flags(self, **flags):\n        \"\"\"\n        Note: For any unmodified flags, update _last_flags with the most recent\n        committed value. Otherwise, for example, this could happen:\n\n            overflow=0\n            instr1 computes overflow=1, updates _last_flags, doesn't commit\n            instr2 updates all flags in _last_flags except overflow (overflow remains 1 in _last_flags)\n            instr2 commits all in _last_flags\n            now overflow=1 even though it should still be 0\n        \"\"\"\n        unupdated_flags = self._last_flags.keys() - flags.keys()\n        for flag in unupdated_flags:\n            flag_name = f'APSR_{flag}'\n            self._last_flags[flag] = self.regfile.read(flag_name)\n        self._last_flags.update(flags)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _shift(cpu, value, _type, amount, carry):\n        assert(cs.arm.ARM_SFT_INVALID < _type <= cs.arm.ARM_SFT_RRX_REG)\n\n        # XXX: Capstone should set the value of an RRX shift to 1, which is\n        # asserted in the manual, but it sets it to 0, so we have to check\n        if _type in (cs.arm.ARM_SFT_RRX, cs.arm.ARM_SFT_RRX_REG) and amount != 1:\n            amount = 1\n\n        elif _type in range(cs.arm.ARM_SFT_ASR_REG, cs.arm.ARM_SFT_RRX_REG + 1):\n            if cpu.mode == cs.CS_MODE_THUMB:\n                src = amount.read()\n            else:\n                src_reg = cpu.instruction.reg_name(amount).upper()\n                src = cpu.regfile.read(src_reg)\n            amount = Operators.EXTRACT(src, 0, 8)\n\n        if amount == 0:\n            return value, carry\n\n        width = cpu.address_bit_size\n\n        if _type in (cs.arm.ARM_SFT_ASR, cs.arm.ARM_SFT_ASR_REG):\n            return ASR_C(value, amount, width)\n        elif _type in (cs.arm.ARM_SFT_LSL, cs.arm.ARM_SFT_LSL_REG):\n            return LSL_C(value, amount, width)\n        elif _type in (cs.arm.ARM_SFT_LSR, cs.arm.ARM_SFT_LSR_REG):\n            return LSR_C(value, amount, width)\n        elif _type in (cs.arm.ARM_SFT_ROR, cs.arm.ARM_SFT_ROR_REG):\n            return ROR_C(value, amount, width)\n        elif _type in (cs.arm.ARM_SFT_RRX, cs.arm.ARM_SFT_RRX_REG):\n            return RRX_C(value, carry, width)\n\n        raise NotImplementedError(\"Bad shift value\")", "response": "Internal function to shift the internal memory of the ARM instruction."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nimplementing the MOV instruction.", "response": "def MOV(cpu, dest, src):\n        \"\"\"\n        Implement the MOV{S} instruction.\n\n        Note: If src operand is PC, temporarily release our logical PC\n        view and conform to the spec, which dictates PC = curr instr + 8\n\n        :param Armv7Operand dest: The destination operand; register.\n        :param Armv7Operand src: The source operand; register or immediate.\n        \"\"\"\n        if cpu.mode == cs.CS_MODE_ARM:\n            result, carry_out = src.read(with_carry=True)\n            dest.write(result)\n            cpu.set_flags(C=carry_out, N=HighBit(result), Z=(result == 0))\n        else:\n            # thumb mode cannot do wonky things to the operand, so no carry calculation\n            result = src.read()\n            dest.write(result)\n            cpu.set_flags(N=HighBit(result), Z=(result == 0))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef MOVT(cpu, dest, src):\n        assert src.type == 'immediate'\n        imm = src.read()\n        low_halfword = dest.read() & Mask(16)\n        dest.write((imm << 16) | low_halfword)", "response": "A MOVT implementation of the AMPL - 1. 1."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef MRC(cpu, coprocessor, opcode1, dest, coprocessor_reg_n, coprocessor_reg_m, opcode2):\n        assert coprocessor.type == 'coprocessor'\n        assert opcode1.type == 'immediate'\n        assert opcode2.type == 'immediate'\n        assert dest.type == 'register'\n        imm_coprocessor = coprocessor.read()\n        imm_opcode1 = opcode1.read()\n        imm_opcode2 = opcode2.read()\n        coprocessor_n_name = coprocessor_reg_n.read()\n        coprocessor_m_name = coprocessor_reg_m.read()\n\n        if 15 == imm_coprocessor:  # MMU\n            if 0 == imm_opcode1:\n                if 13 == coprocessor_n_name:\n                    if 3 == imm_opcode2:\n                        dest.write(cpu.regfile.read('P15_C13'))\n                        return\n        raise NotImplementedError(\"MRC: unimplemented combination of coprocessor, opcode, and coprocessor register\")", "response": "MRC moves to ARM register from the given coprocessor."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nloading double width data from memory.", "response": "def LDRD(cpu, dest1, dest2, src, offset=None):\n        \"\"\"Loads double width data from memory.\"\"\"\n        assert dest1.type == 'register'\n        assert dest2.type == 'register'\n        assert src.type == 'memory'\n        mem1 = cpu.read_int(src.address(), 32)\n        mem2 = cpu.read_int(src.address() + 4, 32)\n        writeback = cpu._compute_writeback(src, offset)\n        dest1.write(mem1)\n        dest2.write(mem2)\n        cpu._cs_hack_ldr_str_writeback(src, offset, writeback)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwrite the contents of two registers to memory.", "response": "def STRD(cpu, src1, src2, dest, offset=None):\n        \"\"\"Writes the contents of two registers to memory.\"\"\"\n        assert src1.type == 'register'\n        assert src2.type == 'register'\n        assert dest.type == 'memory'\n        val1 = src1.read()\n        val2 = src2.read()\n        writeback = cpu._compute_writeback(dest, offset)\n        cpu.write_int(dest.address(), val1, 32)\n        cpu.write_int(dest.address() + 4, val2, 32)\n        cpu._cs_hack_ldr_str_writeback(dest, offset, writeback)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nloads data from memory and add it to the current register.", "response": "def LDREX(cpu, dest, src, offset=None):\n        \"\"\"\n        LDREX loads data from memory.\n        * If the physical address has the shared TLB attribute, LDREX\n          tags the physical address as exclusive access for the current\n          processor, and clears any exclusive access tag for this\n          processor for any other physical address.\n        * Otherwise, it tags the fact that the executing processor has\n          an outstanding tagged physical address.\n\n        :param Armv7Operand dest: the destination register; register\n        :param Armv7Operand src: the source operand: register\n        \"\"\"\n        # TODO: add lock mechanism to underlying memory --GR, 2017-06-06\n        cpu._LDR(dest, src, 32, False, offset)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef STREX(cpu, status, *args):\n        # TODO: implement conditional return with appropriate status --GR, 2017-06-06\n        status.write(0)\n        return cpu._STR(cpu.address_bit_size, *args)", "response": "This function performs a conditional store to memory."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nassembling adds an immediate value to the PC value and writes the result to the destination register.", "response": "def ADR(cpu, dest, src):\n        \"\"\"\n        Address to Register adds an immediate value to the PC value, and writes the result to the destination register.\n\n        :param ARMv7Operand dest: Specifies the destination register.\n        :param ARMv7Operand src:\n            Specifies the label of an instruction or literal data item whose address is to be loaded into\n            <Rd>. The assembler calculates the required value of the offset from the Align(PC,4)\n            value of the ADR instruction to this label.\n        \"\"\"\n        aligned_pc = (cpu.instruction.address + 4) & 0xfffffffc\n        dest.write(aligned_pc + src.read())"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef ADDW(cpu, dest, src, add):\n        aligned_pc = (cpu.instruction.address + 4) & 0xfffffffc\n        if src.type == 'register' and src.reg in ('PC', 'R15'):\n            src = aligned_pc\n        else:\n            src = src.read()\n        dest.write(src + add.read())", "response": "This instruction adds an immediate value to a register value and writes the result to the destination register."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef CBZ(cpu, op, dest):\n        cpu.PC = Operators.ITEBV(cpu.address_bit_size,\n                                 op.read(), cpu.PC, dest.read())", "response": "This function is used to set the instruction PC to the value of the CBZ instruction."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntables Branch Halfword causes a PC - relative forward branch using a table of single halfword offsets.", "response": "def TBH(cpu, dest):\n        \"\"\"\n        Table Branch Halfword causes a PC-relative forward branch using a table of single halfword offsets. A base\n        register provides a pointer to the table, and a second register supplies an index into the table. The branch\n        length is twice the value of the halfword returned from the table.\n\n        :param ARMv7Operand dest: see below; register\n        \"\"\"\n        # Capstone merges the two registers values into one operand, so we need to extract them back\n\n        # Specifies the base register. This contains the address of the table of branch lengths. This\n        # register is allowed to be the PC. If it is, the table immediately follows this instruction.\n        base_addr = dest.get_mem_base_addr()\n        if dest.mem.base in ('PC', 'R15'):\n            base_addr = cpu.PC\n\n        # Specifies the index register. This contains an integer pointing to a halfword within the table.\n        # The offset within the table is twice the value of the index.\n        offset = cpu.read_int(base_addr + dest.get_mem_offset(), 16)\n        offset = Operators.ZEXTEND(offset, cpu.address_bit_size)\n\n        cpu.PC += (offset << 1)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _LDM(cpu, insn_id, base, regs):\n        if cpu.instruction.usermode:\n            raise NotImplementedError(\"Use of the S bit is not supported\")\n\n        increment = insn_id in (cs.arm.ARM_INS_LDM, cs.arm.ARM_INS_LDMIB)\n        after = insn_id in (cs.arm.ARM_INS_LDM, cs.arm.ARM_INS_LDMDA)\n\n        address = base.read()\n\n        for reg in regs:\n            if not after:\n                address += (1 if increment else -1) * (reg.size // 8)\n\n            reg.write(cpu.read_int(address, reg.size))\n            if reg.reg in ('PC', 'R15'):\n                # The general-purpose registers loaded can include the PC. If they do, the word loaded for the PC is\n                # treated as an address and a branch occurs to that address. In ARMv5 and above, bit[0] of the loaded\n                # value determines whether execution continues after this branch in ARM state or in Thumb state, as\n                # though a BX instruction had been executed.\n                cpu._set_mode_by_val(cpu.PC)\n                cpu.PC = cpu.PC & ~1\n\n            if after:\n                address += (1 if increment else -1) * (reg.size // 8)\n\n        if cpu.instruction.writeback:\n            base.writeback(address)", "response": "Load a non - empty subset of registers from the given base register."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _STM(cpu, insn_id, base, regs):\n        if cpu.instruction.usermode:\n            raise NotImplementedError(\"Use of the S bit is not supported\")\n\n        increment = insn_id in (cs.arm.ARM_INS_STM, cs.arm.ARM_INS_STMIB)\n        after = insn_id in (cs.arm.ARM_INS_STM, cs.arm.ARM_INS_STMDA)\n\n        address = base.read()\n\n        for reg in regs:\n            if not after:\n                address += (1 if increment else -1) * (reg.size // 8)\n\n            cpu.write_int(address, reg.read(), reg.size)\n\n            if after:\n                address += (1 if increment else -1) * (reg.size // 8)\n\n        if cpu.instruction.writeback:\n            base.writeback(address)", "response": "Internal function for storing STM instruction."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nnoting on Capstone behavior: - In ARM mode, _SR reg has `rest`, but _SR imm does not, its baked into `op`. - In ARM mode, `lsr r1, r2` will have a `rest[0]` - In Thumb mode, `lsr r1, r2` will have an empty `rest` - In ARM mode, something like `lsr r1, 3` will not have `rest` and op will be the immediate.", "response": "def _SR(cpu, insn_id, dest, op, *rest):\n        \"\"\"\n        Notes on Capstone behavior:\n        - In ARM mode, _SR reg has `rest`, but _SR imm does not, its baked into `op`.\n        - In ARM mode, `lsr r1, r2` will have a `rest[0]`\n        - In Thumb mode, `lsr r1, r2` will have an empty `rest`\n        - In ARM mode, something like `lsr r1, 3` will not have `rest` and op will be\n            the immediate.\n        \"\"\"\n        assert insn_id in (cs.arm.ARM_INS_ASR, cs.arm.ARM_INS_LSL, cs.arm.ARM_INS_LSR)\n\n        if insn_id == cs.arm.ARM_INS_ASR:\n            if rest and rest[0].type == 'immediate':\n                srtype = cs.arm.ARM_SFT_ASR\n            else:\n                srtype = cs.arm.ARM_SFT_ASR_REG\n        elif insn_id == cs.arm.ARM_INS_LSL:\n            if rest and rest[0].type == 'immediate':\n                srtype = cs.arm.ARM_SFT_LSL\n            else:\n                srtype = cs.arm.ARM_SFT_LSL_REG\n        elif insn_id == cs.arm.ARM_INS_LSR:\n            if rest and rest[0].type == 'immediate':\n                srtype = cs.arm.ARM_SFT_LSR\n            else:\n                srtype = cs.arm.ARM_SFT_LSR_REG\n\n        carry = cpu.regfile.read('APSR_C')\n        if rest and rest[0].type == 'register':\n            # FIXME we should make Operand.op private (and not accessible)\n            result, carry = cpu._shift(op.read(), srtype, rest[0].op.reg, carry)\n        elif rest and rest[0].type == 'immediate':\n            amount = rest[0].read()\n            result, carry = cpu._shift(op.read(), srtype, amount, carry)\n        elif cpu.mode == cs.CS_MODE_THUMB:\n            result, carry = cpu._shift(dest.read(), srtype, op, carry)\n        else:\n            result, carry = op.read(with_carry=True)\n        dest.write(result)\n\n        cpu.set_flags(N=HighBit(result), Z=(result == 0), C=carry)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _fix_index(self, index):\n        stop, start = index.stop, index.start\n        if start is None:\n            start = 0\n        if stop is None:\n            stop = len(self)\n        return start, stop", "response": "Fixes the index to be a valid start stop and start index."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _dict_diff(d1, d2):\n    d = {}\n    for key in set(d1).intersection(set(d2)):\n        if d2[key] != d1[key]:\n            d[key] = d2[key]\n    for key in set(d2).difference(set(d1)):\n        d[key] = d2[key]\n    return d", "response": "Returns a dict that includes all the keys in d2 that represent different values in d1."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef context(self):\n        plugin_context_name = str(type(self))\n        if plugin_context_name not in self.manticore.context:\n            self.manticore.context[plugin_context_name] = {}\n        return self.manticore.context[plugin_context_name]", "response": "Return a shared context object"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nlog a finding at specified contract and assembler line.", "response": "def add_finding(self, state, address, pc, finding, at_init, constraint=True):\n        \"\"\"\n        Logs a finding at specified contract and assembler line.\n        :param state: current state\n        :param address: contract address of the finding\n        :param pc: program counter of the finding\n        :param at_init: true if executing the constructor\n        :param finding: textual description of the finding\n        :param constraint: finding is considered reproducible only when constraint is True\n        \"\"\"\n\n        if issymbolic(pc):\n            pc = simplify(pc)\n        if isinstance(pc, Constant):\n            pc = pc.value\n        if not isinstance(pc, int):\n            raise ValueError(\"PC must be a number\")\n        self.get_findings(state).add((address, pc, finding, at_init, constraint))\n        with self.locked_global_findings() as gf:\n            gf.add((address, pc, finding, at_init))\n        #Fixme for ever broken logger\n        logger.warning(finding)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef add_finding_here(self, state, finding, constraint=True):\n        address = state.platform.current_vm.address\n        pc = state.platform.current_vm.pc\n        if isinstance(pc, Constant):\n            pc = pc.value\n        if not isinstance(pc, int):\n            raise ValueError(\"PC must be a number\")\n        at_init = state.platform.current_transaction.sort == 'CREATE'\n        self.add_finding(state, address, pc, finding, at_init, constraint)", "response": "Logs a finding in current contract and assembler line."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _save_current_location(self, state, finding, condition=True):\n        address = state.platform.current_vm.address\n        pc = state.platform.current_vm.pc\n        at_init = state.platform.current_transaction.sort == 'CREATE'\n        location = (address, pc, finding, at_init, condition)\n        hash_id = hashlib.sha1(str(location).encode()).hexdigest()\n        state.context.setdefault('{:s}.locations'.format(self.name), {})[hash_id] = location\n        return hash_id", "response": "Save the current location in the internal locations list and returns a textual id for it."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngetting previously saved location", "response": "def _get_location(self, state, hash_id):\n        \"\"\"\n        Get previously saved location\n        A location is composed of: address, pc, finding, at_init, condition\n        \"\"\"\n        return state.context.setdefault('{:s}.locations'.format(self.name), {})[hash_id]"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _signed_sub_overflow(state, a, b):\n        sub = Operators.SEXTEND(a, 256, 512) - Operators.SEXTEND(b, 256, 512)\n        cond = Operators.OR(sub < -(1 << 255), sub >= (1 << 255))\n        return cond", "response": "Return a condition that can be used to check if the result can be represented\n         in 512 bits and if the result can be represented\n         in 256 bits."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _signed_add_overflow(state, a, b):\n        add = Operators.SEXTEND(a, 256, 512) + Operators.SEXTEND(b, 256, 512)\n        cond = Operators.OR(add < -(1 << 255), add >= (1 << 255))\n        return cond", "response": "Return a condition that can be used to add the result of a signed add operation to the current state."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsigning extend the value to 512 bits and check the result can be represented in 256. Following there is a 32 bit excerpt of this condition: a - b ffffffff bfffffff 80000001 00000000 00000001 3ffffffff 7fffffff ffffffff True True True False True True True bfffffff True True True False False True True 80000001 True True True False False True True 00000000 False False False False False True False 00000001 True False False False False True False ffffffff True True True True True True True 7fffffff True True True False False True False", "response": "def _unsigned_sub_overflow(state, a, b):\n        \"\"\"\n        Sign extend the value to 512 bits and check the result can be represented\n         in 256. Following there is a 32 bit excerpt of this condition:\n\n        a  -  b   ffffffff bfffffff 80000001 00000000 00000001 3ffffffff 7fffffff\n        ffffffff     True     True     True    False     True     True     True\n        bfffffff     True     True     True    False    False     True     True\n        80000001     True     True     True    False    False     True     True\n        00000000    False    False    False    False    False     True    False\n        00000001     True    False    False    False    False     True    False\n        ffffffff     True     True     True     True     True     True     True\n        7fffffff     True     True     True    False    False     True    False\n        \"\"\"\n        cond = Operators.UGT(b, a)\n        return cond"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsigns extend the value to 512 bits and check the result can be represented in 256. Following there is a 32 bit excerpt of this condition: a + b ffffffff bfffffff 80000001 00000000 00000001 3ffffffff 7fffffff ffffffff True True True False True True True bfffffff True True True False False True True 80000001 True True True False False True True 00000000 False False False False False True False 00000001 True False False False False True False ffffffff True True True True True True True 7fffffff True True True False False True False", "response": "def _unsigned_add_overflow(state, a, b):\n        \"\"\"\n        Sign extend the value to 512 bits and check the result can be represented\n         in 256. Following there is a 32 bit excerpt of this condition:\n\n        a  +  b   ffffffff bfffffff 80000001 00000000 00000001 3ffffffff 7fffffff\n        ffffffff     True     True     True    False     True     True     True\n        bfffffff     True     True     True    False    False     True     True\n        80000001     True     True     True    False    False     True     True\n        00000000    False    False    False    False    False     True    False\n        00000001     True    False    False    False    False     True    False\n        ffffffff     True     True     True     True     True     True     True\n        7fffffff     True     True     True    False    False     True    False\n        \"\"\"\n        add = Operators.ZEXTEND(a, 512) + Operators.ZEXTEND(b, 512)\n        cond = Operators.UGE(add, 1 << 256)\n        return cond"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _signed_mul_overflow(state, a, b):\n        mul = Operators.SEXTEND(a, 256, 512) * Operators.SEXTEND(b, 256, 512)\n        cond = Operators.OR(mul < -(1 << 255), mul >= (1 << 255))\n        return cond", "response": "Sign extend the value to 512 bits and check the result can be represented\n         in 256 bits and check the result can be represented\n         in 512 bits and check the result can be represented\n         in 256 bits and return the condition that can be represented by the overflow condition."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsigns extend the value to 512 bits and check the result can be represented in 256. Following there is a 32 bit excerpt of this condition: a * b +00000000000000000 +00000000000000001 +0000000003fffffff +0000000007fffffff +00000000080000001 +000000000bfffffff +000000000ffffffff +0000000000000000 +0000000000000000 +0000000000000000 +0000000000000000 +0000000000000000 +0000000000000000 +0000000000000000 +0000000000000000 +0000000000000001 +0000000000000000 +0000000000000001 +000000003fffffff +000000007fffffff +0000000080000001 +00000000bfffffff +00000000ffffffff +000000003fffffff +0000000000000000 +000000003fffffff *+0fffffff80000001 *+1fffffff40000001 *+1fffffffbfffffff *+2fffffff00000001 *+3ffffffec0000001 +000000007fffffff +0000000000000000 +000000007fffffff *+1fffffff40000001 *+3fffffff00000001 *+3fffffffffffffff *+5ffffffec0000001 *+7ffffffe80000001 +0000000080000001 +0000000000000000 +0000000080000001 *+1fffffffbfffffff *+3fffffffffffffff *+4000000100000001 *+600000003fffffff *+800000007fffffff +00000000bfffffff +0000000000000000 +00000000bfffffff *+2fffffff00000001 *+5ffffffec0000001 *+600000003fffffff *+8ffffffe80000001 *+bffffffe40000001 +00000000ffffffff +0000000000000000 +00000000ffffffff *+3ffffffec0000001 *+7ffffffe80000001 *+800000007fffffff *+bffffffe40000001 *+fffffffe00000001", "response": "def _unsigned_mul_overflow(state, a, b):\n        \"\"\"\n        Sign extend the value to 512 bits and check the result can be represented\n         in 256. Following there is a 32 bit excerpt of this condition:\n\n        a  *  b           +00000000000000000 +00000000000000001 +0000000003fffffff +0000000007fffffff +00000000080000001 +000000000bfffffff +000000000ffffffff\n        +0000000000000000  +0000000000000000  +0000000000000000  +0000000000000000  +0000000000000000  +0000000000000000  +0000000000000000  +0000000000000000\n        +0000000000000001  +0000000000000000  +0000000000000001  +000000003fffffff  +000000007fffffff  +0000000080000001  +00000000bfffffff  +00000000ffffffff\n        +000000003fffffff  +0000000000000000  +000000003fffffff *+0fffffff80000001 *+1fffffff40000001 *+1fffffffbfffffff *+2fffffff00000001 *+3ffffffec0000001\n        +000000007fffffff  +0000000000000000  +000000007fffffff *+1fffffff40000001 *+3fffffff00000001 *+3fffffffffffffff *+5ffffffec0000001 *+7ffffffe80000001\n        +0000000080000001  +0000000000000000  +0000000080000001 *+1fffffffbfffffff *+3fffffffffffffff *+4000000100000001 *+600000003fffffff *+800000007fffffff\n        +00000000bfffffff  +0000000000000000  +00000000bfffffff *+2fffffff00000001 *+5ffffffec0000001 *+600000003fffffff *+8ffffffe80000001 *+bffffffe40000001\n        +00000000ffffffff  +0000000000000000  +00000000ffffffff *+3ffffffec0000001 *+7ffffffe80000001 *+800000007fffffff *+bffffffe40000001 *+fffffffe00000001\n\n        \"\"\"\n        mul = Operators.SEXTEND(a, 256, 512) * Operators.SEXTEND(b, 256, 512)\n        cond = Operators.UGE(mul, 1 << 256)\n        return cond"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _in_user_func(state):\n\n        # If we are already in user function (we cached it) let's just return True\n        in_function = state.context.get('in_function', False)\n        prev_tx_count = state.context.get('prev_tx_count', 0)\n        curr_tx_count = len(state.platform.transactions)\n\n        new_human_tx = prev_tx_count != curr_tx_count\n\n        if in_function and not new_human_tx:\n            return True\n\n        # This is expensive call, so we cache it\n        in_function = len(state.solve_n(state.platform.current_transaction.data[:4], 2)) == 1\n\n        state.context['in_function'] = in_function\n        state.context['prev_tx_count'] = curr_tx_count\n\n        return in_function", "response": "Checks if the current execution is in a user - defined function."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef disassemble_instruction(self, code, pc):\n        return next(self.disasm.disasm(code, pc))", "response": "Disassemble the given code into a new instruction."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn True if an object is tainted.", "response": "def istainted(arg, taint=None):\n    \"\"\"\n    Helper to determine whether an object if tainted.\n    :param arg: a value or Expression\n    :param taint: a regular expression matching a taint value (eg. 'IMPORTANT.*'). If None, this function checks for any taint value.\n    \"\"\"\n\n    if not issymbolic(arg):\n        return False\n    if taint is None:\n        return len(arg.taint) != 0\n    for arg_taint in arg.taint:\n        m = re.match(taint, arg_taint, re.DOTALL | re.IGNORECASE)\n        if m:\n            return True\n    return False"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngives two intervals min1 and max1 and max2 return the intersection interval and the intersection interval.", "response": "def interval_intersection(min1, max1, min2, max2):\n    \"\"\"\n    Given two intervals, (min1, max1) and (min2, max2) return their intersecting interval,\n    or None if they do not overlap.\n    \"\"\"\n    left, right = max(min1, min2), min(max1, max2)\n    if left < right:\n        return left, right\n    return None"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nadds a constraint to the set.", "response": "def add(self, constraint, check=False):\n        \"\"\"\n        Add a constraint to the set\n\n        :param constraint: The constraint to add to the set.\n        :param check: Currently unused.\n        :return:\n        \"\"\"\n        if isinstance(constraint, bool):\n            constraint = BoolConstant(constraint)\n        assert isinstance(constraint, Bool)\n        constraint = simplify(constraint)\n        # If self._child is not None this constraint set has been forked and a\n        # a derived constraintset may be using this. So we can't add any more\n        # constraints to this one. After the child constraintSet is deleted\n        # we regain the ability to add constraints.\n        if self._child is not None:\n            raise Exception('ConstraintSet is frozen')\n\n        if isinstance(constraint, BoolConstant):\n            if not constraint.value:\n                logger.info(\"Adding an impossible constant constraint\")\n                self._constraints = [constraint]\n            else:\n                return\n\n        self._constraints.append(constraint)\n\n        if check:\n            from ...core.smtlib import solver\n            if not solver.check(self):\n                raise ValueError(\"Added an impossible constraint\")"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndeclaring the variable var", "response": "def _declare(self, var):\n        \"\"\" Declare the variable `var` \"\"\"\n        if var.name in self._declarations:\n            raise ValueError('Variable already declared')\n        self._declarations[var.name] = var\n        return var"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef declarations(self):\n        declarations = GetDeclarations()\n        for a in self.constraints:\n            try:\n                declarations.visit(a)\n            except RuntimeError:\n                # TODO: (defunct) move recursion management out of PickleSerializer\n                if sys.getrecursionlimit() >= PickleSerializer.MAX_RECURSION:\n                    raise Exception(f'declarations recursion limit surpassed {PickleSerializer.MAX_RECURSION}, aborting')\n                new_limit = sys.getrecursionlimit() + PickleSerializer.DEFAULT_RECURSION\n                if new_limit <= PickleSerializer.DEFAULT_RECURSION:\n                    sys.setrecursionlimit(new_limit)\n                    return self.declarations\n        return declarations.result", "response": "Returns the variable expressions of this constraint set."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef constraints(self):\n        if self._parent is not None:\n            return tuple(self._constraints) + self._parent.constraints\n        return tuple(self._constraints)", "response": "Returns a tuple of all constraints represented by this and all parent sets."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef is_declared(self, expression_var):\n        if not isinstance(expression_var, Variable):\n            raise ValueError(f'Expression must be a Variable (not a {type(expression_var)})')\n        return any(expression_var is x for x in self.get_declared_variables())", "response": "True if expression_var is declared in this constraint set False otherwise."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nmigrating an expression created for a different constraint set to self. Returns an expression that can be used with this constraintSet All the foreign variables used in the expression are replaced by variables of this constraint set. If the variable was replaced before the replacement is taken from the provided migration map. The migration mapping is updated with new replacements. :param expression: the potentially foreign expression :param name_migration_map: mapping of already migrated variables. maps from string name of foreign variable to its currently existing migrated string name. this is updated during this migration. :return: a migrated expression where all the variables are local. name_migration_map is updated", "response": "def migrate(self, expression, name_migration_map=None):\n        \"\"\" Migrate an expression created for a different constraint set to self.\n            Returns an expression that can be used with this constraintSet\n\n            All the foreign variables used in the expression are replaced by\n            variables of this constraint set. If the variable was replaced before\n            the replacement is taken from the provided migration map.\n\n            The migration mapping is updated with new replacements.\n\n            :param expression: the potentially foreign expression\n            :param name_migration_map: mapping of already migrated variables. maps from string name of foreign variable to its currently existing migrated string name. this is updated during this migration.\n            :return: a migrated expression where all the variables are local. name_migration_map is updated\n\n        \"\"\"\n        if name_migration_map is None:\n            name_migration_map = {}\n\n        #  name_migration_map -> object_migration_map\n        #  Based on the name mapping in name_migration_map build an object to\n        #  object mapping to be used in the replacing of variables\n        #  inv: object_migration_map's keys should ALWAYS be external/foreign\n        #  expressions, and its values should ALWAYS be internal/local expressions\n        object_migration_map = {}\n\n        #List of foreign vars used in expression\n        foreign_vars = itertools.filterfalse(self.is_declared, get_variables(expression))\n        for foreign_var in foreign_vars:\n            # If a variable with the same name was previously migrated\n            if foreign_var.name in name_migration_map:\n                migrated_name = name_migration_map[foreign_var.name]\n                native_var = self.get_variable(migrated_name)\n                assert native_var is not None, \"name_migration_map contains a variable that does not exist in this ConstraintSet\"\n                object_migration_map[foreign_var] = native_var\n            else:\n                # foreign_var was not found in the local declared variables nor\n                # any variable with the same name was previously migrated\n                # let's make a new unique internal name for it\n                migrated_name = foreign_var.name\n                if migrated_name in self._declarations:\n                    migrated_name = self._make_unique_name(f'{foreign_var.name}_migrated')\n                # Create and declare a new variable of given type\n                if isinstance(foreign_var, Bool):\n                    new_var = self.new_bool(name=migrated_name)\n                elif isinstance(foreign_var, BitVec):\n                    new_var = self.new_bitvec(foreign_var.size, name=migrated_name)\n                elif isinstance(foreign_var, Array):\n                    # Note that we are discarding the ArrayProxy encapsulation\n                    new_var = self.new_array(index_max=foreign_var.index_max, index_bits=foreign_var.index_bits, value_bits=foreign_var.value_bits, name=migrated_name).array\n                else:\n                    raise NotImplemented(f\"Unknown expression type {type(var)} encountered during expression migration\")\n                # Update the var to var mapping\n                object_migration_map[foreign_var] = new_var\n                # Update the name to name mapping\n                name_migration_map[foreign_var.name] = new_var.name\n\n        #  Actually replace each appearance of migrated variables by the new ones\n        migrated_expression = replace(expression, object_migration_map)\n        return migrated_expression"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ndeclares a free symbolic boolean in the constraint store .", "response": "def new_bool(self, name=None, taint=frozenset(), avoid_collisions=False):\n        \"\"\" Declares a free symbolic boolean in the constraint store\n            :param name: try to assign name to internal variable representation,\n                         if not unique, a numeric nonce will be appended\n            :param avoid_collisions: potentially avoid_collisions the variable to avoid name collisions if True\n            :return: a fresh BoolVariable\n        \"\"\"\n        if name is None:\n            name = 'B'\n            avoid_collisions = True\n        if avoid_collisions:\n            name = self._make_unique_name(name)\n        if not avoid_collisions and name in self._declarations:\n            raise ValueError(f'Name {name} already used')\n        var = BoolVariable(name, taint=taint)\n        return self._declare(var)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef new_bitvec(self, size, name=None, taint=frozenset(), avoid_collisions=False):\n        if not (size == 1 or size % 8 == 0):\n            raise Exception(f'Invalid bitvec size {size}')\n        if name is None:\n            name = 'BV'\n            avoid_collisions = True\n        if avoid_collisions:\n            name = self._make_unique_name(name)\n        if not avoid_collisions and name in self._declarations:\n            raise ValueError(f'Name {name} already used')\n        var = BitVecVariable(size, name, taint=taint)\n        return self._declare(var)", "response": "Declares a new free symbolic bitvector in the constraint store."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ndeclare a new symbolic array of value_bits long bitvectors in the constraint store.", "response": "def new_array(self, index_bits=32, name=None, index_max=None, value_bits=8, taint=frozenset(), avoid_collisions=False, default=None):\n        \"\"\" Declares a free symbolic array of value_bits long bitvectors in the constraint store.\n            :param index_bits: size in bits for the array indexes one of [32, 64]\n            :param value_bits: size in bits for the array values\n            :param name: try to assign name to internal variable representation,\n                         if not unique, a numeric nonce will be appended\n            :param index_max: upper limit for indexes on this array (#FIXME)\n            :param avoid_collisions: potentially avoid_collisions the variable to avoid name collisions if True\n            :param default: default for not initialized values\n            :return: a fresh ArrayProxy\n        \"\"\"\n        if name is None:\n            name = 'A'\n            avoid_collisions = True\n        if avoid_collisions:\n            name = self._make_unique_name(name)\n        if not avoid_collisions and name in self._declarations:\n            raise ValueError(f'Name {name} already used')\n        var = self._declare(ArrayVariable(index_bits, index_max, value_bits, name, taint=taint))\n        return ArrayProxy(var, default=default)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef train_encoder(X, y, fold_count, encoder):\n    kf = StratifiedKFold(n_splits=fold_count, shuffle=True, random_state=2001)\n    encoder = deepcopy(encoder)  # Because of https://github.com/scikit-learn-contrib/categorical-encoding/issues/106\n    imputer = SimpleImputer(strategy='mean')\n    scaler = StandardScaler()\n    folds = []\n    fit_encoder_time = 0\n    score_encoder_time = 0\n\n    for train_index, test_index in kf.split(X, y):\n        # Split data\n        X_train, X_test = X.iloc[train_index, :].reset_index(drop=True), X.iloc[test_index, :].reset_index(drop=True)\n        y_train, y_test = y[train_index].reset_index(drop=True), y[test_index].reset_index(drop=True)\n\n        # Training\n        start_time = time.time()\n        X_train = encoder.fit_transform(X_train, y_train)\n        fit_encoder_time += time.time() - start_time\n        X_train = imputer.fit_transform(X_train)\n        X_train = scaler.fit_transform(X_train)\n\n        # Testing\n        start_time = time.time()\n        X_test = encoder.transform(X_test)\n        score_encoder_time += time.time() - start_time\n        X_test = imputer.transform(X_test)\n        X_test = scaler.transform(X_test)\n\n        folds.append([X_train, y_train, X_test, y_test])\n\n    return folds, fit_encoder_time/fold_count, score_encoder_time/fold_count", "response": "Train the encoder on the data X y and compute the average fit_encoder_time and score_encoder_time."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ntraining the model on the neural network.", "response": "def train_model(folds, model):\n    \"\"\"\n    Evaluation with:\n      Matthews correlation coefficient: represents thresholding measures\n      AUC: represents ranking measures\n      Brier score: represents calibration measures\n    \"\"\"\n    scores = []\n    fit_model_time = 0      # Sum of all the time spend on fitting the training data, later on normalized\n    score_model_time = 0    # Sum of all the time spend on scoring the testing data, later on normalized\n\n    for X_train, y_train, X_test, y_test in folds:\n        # Training\n        start_time = time.time()\n        with ignore_warnings(category=ConvergenceWarning):  # Yes, neural networks do not always converge\n            model.fit(X_train, y_train)\n        fit_model_time += time.time() - start_time\n        prediction_train_proba = model.predict_proba(X_train)[:, 1]\n        prediction_train = (prediction_train_proba >= 0.5).astype('uint8')\n\n        # Testing\n        start_time = time.time()\n        prediction_test_proba = model.predict_proba(X_test)[:, 1]\n        score_model_time += time.time() - start_time\n        prediction_test = (prediction_test_proba >= 0.5).astype('uint8')\n\n        # When all the predictions are of a single class, we get a RuntimeWarning in matthews_corr\n        with warnings.catch_warnings():\n            warnings.simplefilter(\"ignore\")\n            scores.append([\n                sklearn.metrics.matthews_corrcoef(y_test, prediction_test),\n                sklearn.metrics.matthews_corrcoef(y_train, prediction_train),\n                sklearn.metrics.roc_auc_score(y_test, prediction_test_proba),\n                sklearn.metrics.roc_auc_score(y_train, prediction_train_proba),\n                sklearn.metrics.brier_score_loss(y_test, prediction_test_proba),\n                sklearn.metrics.brier_score_loss(y_train, prediction_train_proba)\n            ])\n\n    return np.mean(scores, axis=0), fit_model_time/len(folds), score_model_time/len(folds)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nfits encoder according to X and y.", "response": "def fit(self, X, y=None, **kwargs):\n        \"\"\"Fit encoder according to X and y.\n\n        Parameters\n        ----------\n\n        X : array-like, shape = [n_samples, n_features]\n            Training vectors, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like, shape = [n_samples]\n            Target values.\n\n        Returns\n        -------\n\n        self : encoder\n            Returns self.\n\n        \"\"\"\n\n        self.base_n_encoder.fit(X, y, **kwargs)\n\n        return self"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef fit(self, X, y=None, **kwargs):\n\n        # if the input dataset isn't already a dataframe, convert it to one (using default column names)\n        # first check the type\n        X = util.convert_input(X)\n\n        self._dim = X.shape[1]\n\n        # if columns aren't passed, just use every string column\n        if self.cols is None:\n            self.cols = util.get_obj_cols(X)\n        else:\n            self.cols = util.convert_cols_to_list(self.cols)\n\n        if self.handle_missing == 'error':\n            if X[self.cols].isnull().any().bool():\n                raise ValueError('Columns to be encoded can not contain null')\n\n        # train an ordinal pre-encoder\n        self.ordinal_encoder = OrdinalEncoder(\n            verbose=self.verbose,\n            cols=self.cols,\n            handle_unknown='value',\n            handle_missing='value'\n        )\n        self.ordinal_encoder = self.ordinal_encoder.fit(X)\n\n        ordinal_mapping = self.ordinal_encoder.category_mapping\n\n        mappings_out = []\n        for switch in ordinal_mapping:\n            values = switch.get('mapping')\n            col = switch.get('col')\n            column_mapping = self.fit_sum_coding(col, values, self.handle_missing, self.handle_unknown)\n            mappings_out.append({'col': switch.get('col'), 'mapping': column_mapping, })\n\n        self.mapping = mappings_out\n\n        X_temp = self.transform(X, override_return_df=True)\n        self.feature_names = X_temp.columns.tolist()\n\n        # drop all output columns with 0 variance.\n        if self.drop_invariant:\n            self.drop_cols = []\n            generated_cols = util.get_generated_cols(X, X_temp, self.cols)\n            self.drop_cols = [x for x in generated_cols if X_temp[x].var() <= 10e-5]\n            try:\n                [self.feature_names.remove(x) for x in self.drop_cols]\n            except KeyError as e:\n                if self.verbose > 0:\n                    print(\"Could not remove column from feature names.\"\n                    \"Not found in generated cols.\\n{}\".format(e))\n\n        return self", "response": "Fit an encoder according to X and y."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nperform the inverse transformation to encoded data.", "response": "def inverse_transform(self, X_in):\n        \"\"\"\n        Perform the inverse transformation to encoded data. Will attempt best case reconstruction, which means\n        it will return nan for handle_missing and handle_unknown settings that break the bijection. We issue\n        warnings when some of those cases occur.\n\n        Parameters\n        ----------\n        X_in : array-like, shape = [n_samples, n_features]\n\n        Returns\n        -------\n        p: array, the same size of X_in\n\n        \"\"\"\n        X = X_in.copy(deep=True)\n\n        # first check the type\n        X = util.convert_input(X)\n\n        if self._dim is None:\n            raise ValueError(\n                'Must train encoder before it can be used to inverse_transform data')\n\n        # then make sure that it is the right size\n        if X.shape[1] != self._dim:\n            if self.drop_invariant:\n                raise ValueError(\"Unexpected input dimension %d, the attribute drop_invariant should \"\n                                 \"set as False when transform data\" % (X.shape[1],))\n            else:\n                raise ValueError('Unexpected input dimension %d, expected %d' % (X.shape[1], self._dim,))\n\n        if not self.cols:\n            return X if self.return_df else X.values\n\n        if self.handle_unknown == 'value':\n            for col in self.cols:\n                if any(X[col] == -1):\n                    warnings.warn(\"inverse_transform is not supported because transform impute \"\n                                  \"the unknown category -1 when encode %s\" % (col,))\n\n        if self.handle_unknown == 'return_nan' and self.handle_missing == 'return_nan':\n            for col in self.cols:\n                if X[col].isnull().any():\n                    warnings.warn(\"inverse_transform is not supported because transform impute \"\n                                  \"the unknown category nan when encode %s\" % (col,))\n\n        for switch in self.mapping:\n            column_mapping = switch.get('mapping')\n            inverse = pd.Series(data=column_mapping.index, index=column_mapping.get_values())\n            X[switch.get('col')] = X[switch.get('col')].map(inverse).astype(switch.get('data_type'))\n\n        return X if self.return_df else X.values"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef ordinal_encoding(X_in, mapping=None, cols=None, handle_unknown='value', handle_missing='value'):\n\n        return_nan_series = pd.Series(data=[np.nan], index=[-2])\n\n        X = X_in.copy(deep=True)\n\n        if cols is None:\n            cols = X.columns.values\n\n        if mapping is not None:\n            mapping_out = mapping\n            for switch in mapping:\n                column = switch.get('col')\n                X[column] = X[column].map(switch['mapping'])\n\n                try:\n                    X[column] = X[column].astype(int)\n                except ValueError as e:\n                    X[column] = X[column].astype(float)\n\n                if handle_unknown == 'value':\n                    X[column].fillna(-1, inplace=True)\n                elif handle_unknown == 'error':\n                    missing = X[column].isnull()\n                    if any(missing):\n                        raise ValueError('Unexpected categories found in column %s' % column)\n\n                if handle_missing == 'return_nan':\n                    X[column] = X[column].map(return_nan_series).where(X[column] == -2, X[column])\n\n        else:\n            mapping_out = []\n            for col in cols:\n\n                nan_identity = np.nan\n\n                if util.is_category(X[col].dtype):\n                    categories = X[col].cat.categories\n                else:\n                    categories = X[col].unique()\n\n                index = pd.Series(categories).fillna(nan_identity).unique()\n\n                data = pd.Series(index=index, data=range(1, len(index) + 1))\n\n                if handle_missing == 'value' and ~data.index.isnull().any():\n                    data.loc[nan_identity] = -2\n                elif handle_missing == 'return_nan':\n                    data.loc[nan_identity] = -2\n\n                mapping_out.append({'col': col, 'mapping': data, 'data_type': X[col].dtype}, )\n\n        return X, mapping_out", "response": "Returns a new dataframe with the classes in the ordinal order."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef transform(self, X, override_return_df=False):\n\n        if self.handle_missing == 'error':\n            if X[self.cols].isnull().any().bool():\n                raise ValueError('Columns to be encoded can not contain null')\n\n        if self._dim is None:\n            raise ValueError(\n                'Must train encoder before it can be used to transform data.')\n\n        # first check the type\n        X = util.convert_input(X)\n\n        # then make sure that it is the right size\n        if X.shape[1] != self._dim:\n            raise ValueError('Unexpected input dimension %d, expected %d' % (\n                X.shape[1], self._dim, ))\n\n        if not self.cols:\n            return X if self.return_df else X.values\n\n        X = self.ordinal_encoder.transform(X)\n\n        if self.handle_unknown == 'error':\n            if X[self.cols].isin([-1]).any().any():\n                raise ValueError('Columns to be encoded can not contain new values')\n\n        X = self.get_dummies(X)\n\n        if self.drop_invariant:\n            for col in self.drop_cols:\n                X.drop(col, 1, inplace=True)\n\n        if self.return_df or override_return_df:\n            return X\n        else:\n            return X.values", "response": "Perform the transformation to new categorical data."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert dummy variable into numerical variables and values represented by the entry in the column mapping.", "response": "def reverse_dummies(self, X, mapping):\n        \"\"\"\n        Convert dummy variable into numerical variables\n\n        Parameters\n        ----------\n        X : DataFrame\n        mapping: list-like\n              Contains mappings of column to be transformed to it's new columns and value represented\n\n        Returns\n        -------\n        numerical: DataFrame\n\n        \"\"\"\n        out_cols = X.columns.values.tolist()\n        mapped_columns = []\n        for switch in mapping:\n            col = switch.get('col')\n            mod = switch.get('mapping')\n            insert_at = out_cols.index(mod.columns[0])\n\n            X.insert(insert_at, col, 0)\n            positive_indexes = mod.index[mod.index > 0]\n            for i in range(positive_indexes.shape[0]):\n                existing_col = mod.columns[i]\n                val = positive_indexes[i]\n                X.loc[X[existing_col] == 1, col] = val\n                mapped_columns.append(existing_col)\n            X.drop(mod.columns, axis=1, inplace=True)\n            out_cols = X.columns.values.tolist()\n\n        return X"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_cars_data():\n\n    df = pd.read_csv('source_data/cars/car.data.txt')\n    X = df.reindex(columns=[x for x in df.columns.values if x != 'class'])\n    y = df.reindex(columns=['class'])\n    y = preprocessing.LabelEncoder().fit_transform(y.values.reshape(-1, ))\n\n    mapping = [\n        {'col': 'buying', 'mapping': [('vhigh', 0), ('high', 1), ('med', 2), ('low', 3)]},\n        {'col': 'maint', 'mapping': [('vhigh', 0), ('high', 1), ('med', 2), ('low', 3)]},\n        {'col': 'doors', 'mapping': [('2', 0), ('3', 1), ('4', 2), ('5more', 3)]},\n        {'col': 'persons', 'mapping': [('2', 0), ('4', 1), ('more', 2)]},\n        {'col': 'lug_boot', 'mapping': [('small', 0), ('med', 1), ('big', 2)]},\n        {'col': 'safety', 'mapping': [('high', 0), ('med', 1), ('low', 2)]},\n    ]\n\n    return X, y, mapping", "response": "Load the cars dataset and split it into X and y columns and call label encoder to get an integer y column."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_splice_data():\n\n    df = pd.read_csv('source_data/splice/splice.csv')\n    X = df.reindex(columns=[x for x in df.columns.values if x != 'class'])\n    X['dna'] = X['dna'].map(lambda x: list(str(x).strip()))\n    for idx in range(60):\n        X['dna_%d' % (idx, )] = X['dna'].map(lambda x: x[idx])\n    del X['dna']\n\n    y = df.reindex(columns=['class'])\n    y = preprocessing.LabelEncoder().fit_transform(y.values.reshape(-1, ))\n\n    # this data is truly categorical, with no known concept of ordering\n    mapping = None\n\n    return X, y, mapping", "response": "Load the mushroom dataset split it into X and y and call label encoder to get an integer y column."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting basen code to integers.", "response": "def basen_to_integer(self, X, cols, base):\n        \"\"\"\n        Convert basen code as integers.\n\n        Parameters\n        ----------\n        X : DataFrame\n            encoded data\n        cols : list-like\n            Column names in the DataFrame that be encoded\n        base : int\n            The base of transform\n\n        Returns\n        -------\n        numerical: DataFrame\n\n        \"\"\"\n        out_cols = X.columns.values.tolist()\n\n        for col in cols:\n            col_list = [col0 for col0 in out_cols if str(col0).startswith(str(col))]\n            insert_at = out_cols.index(col_list[0])\n\n            if base == 1:\n                value_array = np.array([int(col0.split('_')[-1]) for col0 in col_list])\n            else:\n                len0 = len(col_list)\n                value_array = np.array([base ** (len0 - 1 - i) for i in range(len0)])\n            X.insert(insert_at, col, np.dot(X[col_list].values, value_array.T))\n            X.drop(col_list, axis=1, inplace=True)\n            out_cols = X.columns.values.tolist()\n\n        return X"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef col_transform(self, col, digits):\n\n        if col is None or float(col) < 0.0:\n            return None\n        else:\n            col = self.number_to_base(int(col), self.base, digits)\n            if len(col) == digits:\n                return col\n            else:\n                return [0 for _ in range(digits - len(col))] + col", "response": "The lambda body to transform the column values\n           "}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef fit(self, X, y=None, **kwargs):\n\n        # first check the type\n        X = util.convert_input(X)\n\n        self._dim = X.shape[1]\n\n        # if columns aren't passed, just use every string column\n        if self.cols is None:\n            self.cols = util.get_obj_cols(X)\n        else:\n            self.cols = util.convert_cols_to_list(self.cols)\n\n        X_temp = self.transform(X, override_return_df=True)\n        self.feature_names = X_temp.columns.tolist()\n\n        # drop all output columns with 0 variance.\n        if self.drop_invariant:\n            self.drop_cols = []\n            generated_cols = util.get_generated_cols(X, X_temp, self.cols)\n            self.drop_cols = [x for x in generated_cols if X_temp[x].var() <= 10e-5]\n            try:\n                [self.feature_names.remove(x) for x in self.drop_cols]\n            except KeyError as e:\n                if self.verbose > 0:\n                    print(\"Could not remove column from feature names.\"\n                          \"Not found in generated cols.\\n{}\".format(e))\n\n        return self", "response": "Fit the encoder according to X and y."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef transform(self, X, override_return_df=False):\n\n        if self._dim is None:\n            raise ValueError('Must train encoder before it can be used to transform data.')\n\n        # first check the type\n        X = util.convert_input(X)\n\n        # then make sure that it is the right size\n        if X.shape[1] != self._dim:\n            raise ValueError('Unexpected input dimension %d, expected %d' % (X.shape[1], self._dim, ))\n\n        if not self.cols:\n            return X\n\n        X = self.hashing_trick(X, hashing_method=self.hash_method, N=self.n_components, cols=self.cols)\n\n        if self.drop_invariant:\n            for col in self.drop_cols:\n                X.drop(col, 1, inplace=True)\n\n        if self.return_df or override_return_df:\n            return X\n        else:\n            return X.values", "response": "Perform the transformation to new categorical data."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nperforms the transformation to new categorical data.", "response": "def transform(self, X, y=None, override_return_df=False):\n        \"\"\"Perform the transformation to new categorical data.\n\n        Parameters\n        ----------\n\n        X : array-like, shape = [n_samples, n_features]\n        y : array-like, shape = [n_samples] when transform by leave one out\n            None, when transform without target information (such as transform test set)\n\n\n\n        Returns\n        -------\n\n        p : array, shape = [n_samples, n_numeric + N]\n            Transformed values with encoding applied.\n\n        \"\"\"\n\n        if self.handle_missing == 'error':\n            if X[self.cols].isnull().any().bool():\n                raise ValueError('Columns to be encoded can not contain null')\n\n        if self._dim is None:\n            raise ValueError('Must train encoder before it can be used to transform data.')\n\n        # unite the input into pandas types\n        X = util.convert_input(X)\n\n        # then make sure that it is the right size\n        if X.shape[1] != self._dim:\n            raise ValueError('Unexpected input dimension %d, expected %d' % (X.shape[1], self._dim,))\n\n        # if we are encoding the training data, we have to check the target\n        if y is not None:\n            y = util.convert_input_vector(y, X.index).astype(float)\n            if X.shape[0] != y.shape[0]:\n                raise ValueError(\"The length of X is \" + str(X.shape[0]) + \" but length of y is \" + str(y.shape[0]) + \".\")\n\n        if not self.cols:\n            return X\n        X = self.transform_leave_one_out(\n            X, y,\n            mapping=self.mapping\n        )\n\n        if self.drop_invariant:\n            for col in self.drop_cols:\n                X.drop(col, 1, inplace=True)\n\n        if self.return_df or override_return_df:\n            return X\n        else:\n            return X.values"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ntransform leave - one - out encoding uses a single column of floats to represent the means of the target variables.", "response": "def transform_leave_one_out(self, X_in, y, mapping=None):\n        \"\"\"\n        Leave one out encoding uses a single column of floats to represent the means of the target variables.\n        \"\"\"\n        X = X_in.copy(deep=True)\n        random_state_ = check_random_state(self.random_state)\n\n        # Prepare the data\n        if y is not None:\n            # Convert bools to numbers (the target must be summable)\n            y = y.astype('double')\n\n            # Cumsum and cumcount do not work nicely with None.\n            # This is a terrible workaround that will fail, when the\n            # categorical input contains -999.9\n            for cat_col in X.select_dtypes('category').columns.values:\n                X[cat_col] = X[cat_col].cat.add_categories(-999.9)\n            X = X.fillna(-999.9)\n\n        for col, colmap in mapping.items():\n            level_notunique = colmap['count'] > 1\n\n            unique_train = colmap.index\n            unseen_values = pd.Series([x for x in X_in[col].unique() if x not in unique_train])\n\n            is_nan = X_in[col].isnull()\n            is_unknown_value = X_in[col].isin(unseen_values.dropna())\n\n            if self.handle_unknown == 'error' and is_unknown_value.any():\n                raise ValueError('Columns to be encoded can not contain new values')\n\n            if y is None:    # Replace level with its mean target; if level occurs only once, use global mean\n                level_means = ((colmap['sum'] + self._mean) / (colmap['count'] + 1)).where(level_notunique, self._mean)\n                X[col] = X[col].map(level_means)\n            else:\n                # Simulation of CatBoost implementation, which calculates leave-one-out on the fly.\n                # The nice thing about this is that it helps to prevent overfitting. The bad thing\n                # is that CatBoost uses many iterations over the data. But we run just one iteration.\n                # Still, it works better than leave-one-out without any noise.\n                # See:\n                #   https://tech.yandex.com/catboost/doc/dg/concepts/algorithm-main-stages_cat-to-numberic-docpage/\n                temp = y.groupby(X[col]).agg(['cumsum', 'cumcount'])\n                X[col] = (temp['cumsum'] - y + self._mean) / (temp['cumcount'] + 1)\n\n            if self.handle_unknown == 'value':\n                X.loc[is_unknown_value, col] = self._mean\n            elif self.handle_unknown == 'return_nan':\n                X.loc[is_unknown_value, col] = np.nan\n\n            if self.handle_missing == 'value':\n                X.loc[is_nan & unseen_values.isnull().any(), col] = self._mean\n            elif self.handle_missing == 'return_nan':\n                X.loc[is_nan, col] = np.nan\n\n            if self.sigma is not None and y is not None:\n                X[col] = X[col] * random_state_.normal(1., self.sigma, X[col].shape[0])\n\n        return X"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_obj_cols(df):\n    obj_cols = []\n    for idx, dt in enumerate(df.dtypes):\n        if dt == 'object' or is_category(dt):\n            obj_cols.append(df.columns.values[idx])\n\n    return obj_cols", "response": "Returns names of object columns in the DataFrame."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef convert_input(X):\n    if not isinstance(X, pd.DataFrame):\n        if isinstance(X, list):\n            X = pd.DataFrame(X)\n        elif isinstance(X, (np.generic, np.ndarray)):\n            X = pd.DataFrame(X)\n        elif isinstance(X, csr_matrix):\n            X = pd.DataFrame(X.todense())\n        elif isinstance(X, pd.Series):\n            X = pd.DataFrame(X)\n        else:\n            raise ValueError('Unexpected input type: %s' % (str(type(X))))\n\n        X = X.apply(lambda x: pd.to_numeric(x, errors='ignore'))\n\n    return X", "response": "Convert input to a DataFrame."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nconverting input vector into a Unite target data type.", "response": "def convert_input_vector(y, index):\n    \"\"\"\n    Unite target data type into a Series.\n    If the target is a Series or a DataFrame, we preserve its index.\n    But if the target does not contain index attribute, we use the index from the argument.\n    \"\"\"\n    if y is None:\n        return None\n    if isinstance(y, pd.Series):\n        return y\n    elif isinstance(y, np.ndarray):\n        if len(np.shape(y))==1:  # vector\n            return pd.Series(y, name='target', index=index)\n        elif len(np.shape(y))==2 and np.shape(y)[0]==1:  # single row in a matrix\n            return pd.Series(y[0, :], name='target', index=index)\n        elif len(np.shape(y))==2 and np.shape(y)[1]==1:  # single column in a matrix\n            return pd.Series(y[:, 0], name='target', index=index)\n        else:\n            raise ValueError('Unexpected input shape: %s' % (str(np.shape(y))))\n    elif np.isscalar(y):\n        return pd.Series([y], name='target', index=index)\n    elif isinstance(y, list):\n        if len(y)==0 or (len(y)>0 and not isinstance(y[0], list)): # empty list or a vector\n            return pd.Series(y, name='target', index=index)\n        elif len(y)>0 and isinstance(y[0], list) and len(y[0])==1: # single row in a matrix\n            flatten = lambda y: [item for sublist in y for item in sublist]\n            return pd.Series(flatten(y), name='target', index=index)\n        elif len(y)==1 and isinstance(y[0], list): # single column in a matrix\n            return pd.Series(y[0], name='target', index=index)\n        else:\n            raise ValueError('Unexpected input shape')\n    elif isinstance(y, pd.DataFrame):\n        if len(list(y))==0: # empty DataFrame\n            return pd.Series(y, name='target')\n        if len(list(y))==1: # a single column\n            return y.iloc[:, 0]\n        else:\n            raise ValueError('Unexpected input shape: %s' % (str(y.shape)))\n    else:\n        return pd.Series(y, name='target', index=index)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a list of the generated columns.", "response": "def get_generated_cols(X_original, X_transformed, to_transform):\n    \"\"\"\n    Returns a list of the generated/transformed columns.\n\n    Arguments:\n        X_original: df\n            the original (input) DataFrame.\n        X_transformed: df\n            the transformed (current) DataFrame.\n        to_transform: [str]\n            a list of columns that were transformed (as in the original DataFrame), commonly self.cols.\n\n    Output:\n        a list of columns that were transformed (as in the current DataFrame).\n    \"\"\"\n    original_cols = list(X_original.columns)\n\n    if len(to_transform) > 0:\n        [original_cols.remove(c) for c in to_transform]\n\n    current_cols = list(X_transformed.columns)\n    if len(original_cols) > 0:\n        [current_cols.remove(c) for c in original_cols]\n\n    return current_cols"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nfit the encoder according to X and y.", "response": "def fit(self, X, y, **kwargs):\n        \"\"\"Fit encoder according to X and y.\n\n        Parameters\n        ----------\n        X : array-like, shape = [n_samples, n_features]\n            Training vectors, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like, shape = [n_samples]\n            Target values.\n\n        Returns\n        -------\n        self : encoder\n            Returns self.\n\n        \"\"\"\n\n        # unite the input into pandas types\n        X = util.convert_input(X)\n        y = util.convert_input_vector(y, X.index)\n\n        if X.shape[0] != y.shape[0]:\n            raise ValueError(\"The length of X is \" + str(X.shape[0]) + \" but length of y is \" + str(y.shape[0]) + \".\")\n\n        self._dim = X.shape[1]\n\n        # if columns aren't passed, just use every string column\n        if self.cols is None:\n            self.cols = util.get_obj_cols(X)\n        else:\n            self.cols = util.convert_cols_to_list(self.cols)\n\n        if self.handle_missing == 'error':\n            if X[self.cols].isnull().any().bool():\n                raise ValueError('Columns to be encoded can not contain null')\n\n        self.ordinal_encoder = OrdinalEncoder(\n            verbose=self.verbose,\n            cols=self.cols,\n            handle_unknown='value',\n            handle_missing='value'\n        )\n        self.ordinal_encoder = self.ordinal_encoder.fit(X)\n        X_ordinal = self.ordinal_encoder.transform(X)\n        self.mapping = self.fit_target_encoding(X_ordinal, y)\n        \n        X_temp = self.transform(X, override_return_df=True)\n        self.feature_names = list(X_temp.columns)\n\n        if self.drop_invariant:\n            self.drop_cols = []\n            X_temp = self.transform(X)\n            generated_cols = util.get_generated_cols(X, X_temp, self.cols)\n            self.drop_cols = [x for x in generated_cols if X_temp[x].var() <= 10e-5]\n            try:\n                [self.feature_names.remove(x) for x in self.drop_cols]\n            except KeyError as e:\n                if self.verbose > 0:\n                    print(\"Could not remove column from feature names.\"\n                    \"Not found in generated cols.\\n{}\".format(e))\n\n        return self"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef transform_leave_one_out(self, X_in, y, mapping=None):\n\n        X = X_in.copy(deep=True)\n        random_state_ = check_random_state(self.random_state)\n\n        for col, colmap in mapping.items():\n            level_notunique = colmap['count'] > 1\n\n            unique_train = colmap.index\n            unseen_values = pd.Series([x for x in X[col].unique() if x not in unique_train])\n\n            is_nan = X[col].isnull()\n            is_unknown_value = X[col].isin(unseen_values.dropna())\n\n            if self.handle_unknown == 'error' and is_unknown_value.any():\n                raise ValueError('Columns to be encoded can not contain new values')\n\n            if y is None:    # Replace level with its mean target; if level occurs only once, use global mean\n                level_means = (colmap['sum'] / colmap['count']).where(level_notunique, self._mean)\n                X[col] = X[col].map(level_means)\n            else:            # Replace level with its mean target, calculated excluding this row's target\n                # The y (target) mean for this level is normally just the sum/count;\n                # excluding this row's y, it's (sum - y) / (count - 1)\n                level_means = (X[col].map(colmap['sum']) - y) / (X[col].map(colmap['count']) - 1)\n                # The 'where' fills in singleton levels (count = 1 -> div by 0) with the global mean\n                X[col] = level_means.where(X[col].map(colmap['count'][level_notunique]).notnull(), self._mean)\n\n            if self.handle_unknown == 'value':\n                X.loc[is_unknown_value, col] = self._mean\n            elif self.handle_unknown == 'return_nan':\n                X.loc[is_unknown_value, col] = np.nan\n\n            if self.handle_missing == 'value':\n                X.loc[is_nan & unseen_values.isnull().any(), col] = self._mean\n            elif self.handle_missing == 'return_nan':\n                X.loc[is_nan, col] = np.nan\n\n            if self.sigma is not None and y is not None:\n                X[col] = X[col] * random_state_.normal(1., self.sigma, X[col].shape[0])\n\n        return X", "response": "Transform leave one out encoding uses a single column of floats to represent the means of the target variables."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef score_models(clf, X, y, encoder, runs=1):\n\n    scores = []\n\n    X_test = None\n    for _ in range(runs):\n        X_test = encoder().fit_transform(X, y)\n\n        # Some models, like logistic regression, like normalized features otherwise they underperform and/or take a long time to converge.\n        # To be rigorous, we should have trained the normalization on each fold individually via pipelines.\n        # See grid_search_example to learn how to do it.\n        X_test = StandardScaler().fit_transform(X_test)\n\n        scores.append(cross_validate(clf, X_test, y, n_jobs=1, cv=5)['test_score'])\n        gc.collect()\n\n    scores = [y for z in [x for x in scores] for y in z]\n\n    return float(np.mean(scores)), float(np.std(scores)), scores, X_test.shape[1]", "response": "Takes in a classifier that supports multiclass classification and X and a y and returns a cross validation score."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef secho(message, **kwargs):\n    # If colors are disabled, remove any color or other style data\n    # from keyword arguments.\n    if not settings.color:\n        for key in ('fg', 'bg', 'bold', 'blink'):\n            kwargs.pop(key, None)\n\n    # Okay, now call click.secho normally.\n    return click.secho(message, **kwargs)", "response": "Wrapper around click. secho that disables any coloring being used."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nassociates a notification template with this job template.", "response": "def associate_notification_template(self, job_template,\n                                        notification_template, status):\n        \"\"\"Associate a notification template from this job template.\n\n        =====API DOCS=====\n        Associate a notification template from this job template.\n\n        :param job_template: The job template to associate to.\n        :type job_template: str\n        :param notification_template: The notification template to be associated.\n        :type notification_template: str\n        :param status: type of notification this notification template should be associated to.\n        :type status: str\n        :returns: Dictionary of only one key \"changed\", which indicates whether the association succeeded.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        return self._assoc('notification_templates_%s' % status,\n                           job_template, notification_template)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndisassociating a notification template from this job template.", "response": "def disassociate_notification_template(self, job_template,\n                                           notification_template, status):\n        \"\"\"Disassociate a notification template from this job template.\n\n        =====API DOCS=====\n        Disassociate a notification template from this job template.\n\n        :param job_template: The job template to disassociate from.\n        :type job_template: str\n        :param notification_template: The notification template to be disassociated.\n        :type notification_template: str\n        :param status: type of notification this notification template should be disassociated from.\n        :type status: str\n        :returns: Dictionary of only one key \"changed\", which indicates whether the disassociation succeeded.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        return self._disassoc('notification_templates_%s' % status,\n                              job_template, notification_template)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef callback(self, pk=None, host_config_key='', extra_vars=None):\n        url = self.endpoint + '%s/callback/' % pk\n        if not host_config_key:\n            host_config_key = client.get(url).json()['host_config_key']\n        post_data = {'host_config_key': host_config_key}\n        if extra_vars:\n            post_data['extra_vars'] = parser.process_extra_vars(list(extra_vars), force_json=True)\n        r = client.post(url, data=post_data, auth=None)\n        if r.status_code == 201:\n            return {'changed': True}", "response": "Contact Tower and request a configuration update using this job template."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef jt_aggregate(func, is_create=False, has_pk=False):\n    def helper(kwargs, obj):\n        \"\"\"The helper function preceding actual function that aggregates\n        unified jt fields.\n        \"\"\"\n        unified_job_template = None\n        for item in UNIFIED_JT:\n            if kwargs.get(item, None) is not None:\n                jt_id = kwargs.pop(item)\n                if unified_job_template is None:\n                    unified_job_template = (item, jt_id)\n                else:\n                    raise exc.UsageError(\n                        'More than one unified job template fields provided, '\n                        'please tighten your criteria.'\n                    )\n        if unified_job_template is not None:\n            kwargs['unified_job_template'] = unified_job_template[1]\n            obj.identity = tuple(list(obj.identity) + ['unified_job_template'])\n            return '/'.join([UNIFIED_JT[unified_job_template[0]],\n                             str(unified_job_template[1]), 'schedules/'])\n        elif is_create:\n            raise exc.UsageError('You must provide exactly one unified job'\n                                 ' template field during creation.')\n\n    def decorator_without_pk(obj, *args, **kwargs):\n        old_endpoint = obj.endpoint\n        new_endpoint = helper(kwargs, obj)\n        if is_create:\n            obj.endpoint = new_endpoint\n        result = func(obj, *args, **kwargs)\n        obj.endpoint = old_endpoint\n        return result\n\n    def decorator_with_pk(obj, pk=None, *args, **kwargs):\n        old_endpoint = obj.endpoint\n        new_endpoint = helper(kwargs, obj)\n        if is_create:\n            obj.endpoint = new_endpoint\n        result = func(obj, pk=pk, *args, **kwargs)\n        obj.endpoint = old_endpoint\n        return result\n\n    decorator = decorator_with_pk if has_pk else decorator_without_pk\n    for item in CLICK_ATTRS:\n        setattr(decorator, item, getattr(func, item, []))\n    decorator.__doc__ = func.__doc__\n\n    return decorator", "response": "Decorator to aggregate unified_jt - related fields."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef lookup_stdout(self, pk=None, start_line=None, end_line=None,\n                      full=True):\n        \"\"\"\n        Internal method that lies to our `monitor` method by returning\n        a scorecard for the workflow job where the standard out\n        would have been expected.\n        \"\"\"\n        uj_res = get_resource('unified_job')\n        # Filters\n        #  - limit search to jobs spawned as part of this workflow job\n        #  - order in the order in which they should add to the list\n        #  - only include final job states\n        query_params = (('unified_job_node__workflow_job', pk),\n                        ('order_by', 'finished'),\n                        ('status__in', 'successful,failed,error'))\n        jobs_list = uj_res.list(all_pages=True, query=query_params)\n        if jobs_list['count'] == 0:\n            return ''\n\n        return_content = ResSubcommand(uj_res)._format_human(jobs_list)\n        lines = return_content.split('\\n')\n        if not full:\n            lines = lines[:-1]\n\n        N = len(lines)\n        start_range = start_line\n        if start_line is None:\n            start_range = 0\n        elif start_line > N:\n            start_range = N\n\n        end_range = end_line\n        if end_line is None or end_line > N:\n            end_range = N\n\n        lines = lines[start_range:end_range]\n        return_content = '\\n'.join(lines)\n        if len(lines) > 0:\n            return_content += '\\n'\n\n        return return_content", "response": "This method returns the standard output of a workflow job."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nlaunching a new workflow job based on a workflow job template.", "response": "def launch(self, workflow_job_template=None, monitor=False, wait=False,\n               timeout=None, extra_vars=None, **kwargs):\n        \"\"\"Launch a new workflow job based on a workflow job template.\n\n        Creates a new workflow job in Ansible Tower, starts it, and\n        returns back an ID in order for its status to be monitored.\n\n        =====API DOCS=====\n        Launch a new workflow job based on a workflow job template.\n\n        :param workflow_job_template: Primary key or name of the workflow job template to launch new job.\n        :type workflow_job_template: str\n        :param monitor: Flag that if set, immediately calls ``monitor`` on the newly launched workflow job rather\n                        than exiting with a success.\n        :type monitor: bool\n        :param wait: Flag that if set, monitor the status of the workflow job, but do not print while job is\n                     in progress.\n        :type wait: bool\n        :param timeout: If provided with ``monitor`` flag set, this attempt will time out after the given number\n                        of seconds.\n        :type timeout: int\n        :param extra_vars: yaml formatted texts that contains extra variables to pass on.\n        :type extra_vars: array of strings\n        :param `**kwargs`: Fields needed to create and launch a workflow job.\n        :returns: Result of subsequent ``monitor`` call if ``monitor`` flag is on; Result of subsequent ``wait``\n                  call if ``wait`` flag is on; loaded JSON output of the job launch if none of the two flags are on.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        if extra_vars is not None and len(extra_vars) > 0:\n            kwargs['extra_vars'] = parser.process_extra_vars(extra_vars)\n\n        debug.log('Launching the workflow job.', header='details')\n        self._pop_none(kwargs)\n        post_response = client.post('workflow_job_templates/{0}/launch/'.format(\n            workflow_job_template), data=kwargs).json()\n\n        workflow_job_id = post_response['id']\n        post_response['changed'] = True\n\n        if monitor:\n            return self.monitor(workflow_job_id, timeout=timeout)\n        elif wait:\n            return self.wait(workflow_job_id, timeout=timeout)\n\n        return post_response"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_args(self, ctx, args):\n        if not args and self.no_args_is_help and not ctx.resilient_parsing:\n            click.echo(ctx.get_help())\n            ctx.exit()\n        return super(ActionSubcommand, self).parse_args(ctx, args)", "response": "Parse arguments sent to this command."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmonkeying - patch click s format_options method to support option categorization.", "response": "def format_options(self, ctx, formatter):\n        \"\"\"Monkey-patch click's format_options method to support option categorization.\n        \"\"\"\n        field_opts = []\n        global_opts = []\n        local_opts = []\n        other_opts = []\n        for param in self.params:\n            if param.name in SETTINGS_PARMS:\n                opts = global_opts\n            elif getattr(param, 'help', None) and param.help.startswith('[FIELD]'):\n                opts = field_opts\n                param.help = param.help[len('[FIELD]'):]\n            else:\n                opts = local_opts\n            rv = param.get_help_record(ctx)\n            if rv is None:\n                continue\n            else:\n                opts.append(rv)\n\n        if self.add_help_option:\n            help_options = self.get_help_option_names(ctx)\n            if help_options:\n                other_opts.append([join_options(help_options)[0], 'Show this message and exit.'])\n\n        if field_opts:\n            with formatter.section('Field Options'):\n                formatter.write_dl(field_opts)\n        if local_opts:\n            with formatter.section('Local Options'):\n                formatter.write_dl(local_opts)\n        if global_opts:\n            with formatter.section('Global Options'):\n                formatter.write_dl(global_opts)\n        if other_opts:\n            with formatter.section('Other Options'):\n                formatter.write_dl(other_opts)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef list(self, **kwargs):\n        self.custom_category = kwargs.get('category', 'all')\n        try:\n            result = super(Resource, self).list(**kwargs)\n        except exc.NotFound as e:\n            categories = map(\n                lambda category: category['slug'],\n                client.get('/settings/').json()['results']\n            )\n            e.message = '%s is not a valid category.  Choose from [%s]' % (\n                kwargs['category'],\n                ', '.join(categories)\n            )\n            raise e\n        finally:\n            self.custom_category = None\n        return {\n            'results': [{'id': k, 'value': v} for k, v in result.items()]\n        }", "response": "Return a list of objects."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nretrieve a single Tower setting from the API.", "response": "def get(self, pk):\n        \"\"\"Return one and exactly one object\n\n        =====API DOCS=====\n        Return one and exactly one Tower setting.\n\n        :param pk: Primary key of the Tower setting to retrieve\n        :type pk: int\n        :returns: loaded JSON of the retrieved Tower setting object.\n        :rtype: dict\n        :raises tower_cli.exceptions.NotFound: When no specified Tower setting exists.\n\n        =====API DOCS=====\n        \"\"\"\n        # The Tower API doesn't provide a mechanism for retrieving a single\n        # setting value at a time, so fetch them all and filter\n        try:\n            return next(s for s in self.list()['results'] if s['id'] == pk)\n        except StopIteration:\n            raise exc.NotFound('The requested object could not be found.')"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef modify(self, setting, value):\n        prev_value = new_value = self.get(setting)['value']\n        answer = OrderedDict()\n        encrypted = '$encrypted$' in six.text_type(prev_value)\n\n        if encrypted or six.text_type(prev_value) != six.text_type(value):\n            if setting == 'LICENSE':\n                r = client.post('/config/',\n                                data=self.coerce_type(setting, value))\n                new_value = r.json()\n            else:\n                r = client.patch(\n                    self.endpoint,\n                    data={setting: self.coerce_type(setting, value)}\n                )\n                new_value = r.json()[setting]\n            answer.update(r.json())\n\n        changed = encrypted or (prev_value != new_value)\n\n        answer.update({\n            'changed': changed,\n            'id': setting,\n            'value': new_value,\n        })\n        return answer", "response": "Modify an existing object."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nremoves default values from kwargs.", "response": "def _pop_none(self, kwargs):\n        \"\"\"Remove default values (anything where the value is None). click is unfortunately bad at the way it\n        sends through unspecified defaults.\"\"\"\n        for key, value in copy(kwargs).items():\n            # options with multiple=True return a tuple\n            if value is None or value == ():\n                kwargs.pop(key)\n            if hasattr(value, 'read'):\n                kwargs[key] = value.read()"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _lookup(self, fail_on_missing=False, fail_on_found=False, include_debug_header=True, **kwargs):\n        read_params = {}\n        for field_name in self.identity:\n            if field_name in kwargs:\n                read_params[field_name] = kwargs[field_name]\n        if 'id' in self.identity and len(self.identity) == 1:\n            return {}\n        if not read_params:\n            raise exc.BadRequest('Cannot reliably determine which record to write. Include an ID or unique '\n                                 'fields.')\n        try:\n            existing_data = self.get(include_debug_header=include_debug_header, **read_params)\n            if fail_on_found:\n                raise exc.Found('A record matching %s already exists, and you requested a failure in that case.' %\n                                read_params)\n            return existing_data\n        except exc.NotFound:\n            if fail_on_missing:\n                raise exc.NotFound('A record matching %s does not exist, and you requested a failure in that case.' %\n                                   read_params)\n            return {}", "response": "This method is used to perform a lookup of the resource record details."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert next and previous from URLs to integers", "response": "def _convert_pagenum(self, kwargs):\n        \"\"\"\n        Convert next and previous from URLs to integers\n        \"\"\"\n        for key in ('next', 'previous'):\n            if not kwargs.get(key):\n                continue\n            match = re.search(r'page=(?P<num>[\\d]+)', kwargs[key])\n            if match is None and key == 'previous':\n                kwargs[key] = 1\n                continue\n            kwargs[key] = int(match.groupdict()['num'])"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef read(self, pk=None, fail_on_no_results=False, fail_on_multiple_results=False, **kwargs):\n        # Piece together the URL we will be hitting.\n        url = self.endpoint\n        if pk:\n            url += '%s/' % pk\n\n        # Pop the query parameter off of the keyword arguments; it will\n        # require special handling (below).\n        queries = kwargs.pop('query', [])\n\n        # Remove default values (anything where the value is None).\n        self._pop_none(kwargs)\n\n        # Remove fields that are specifically excluded from lookup\n        for field in self.fields:\n            if field.no_lookup and field.name in kwargs:\n                kwargs.pop(field.name)\n\n        # If queries were provided, process them.\n        params = list(kwargs.items())\n        for query in queries:\n            params.append((query[0], query[1]))\n\n        # Make the request to the Ansible Tower API.\n        r = client.get(url, params=params)\n        resp = r.json()\n\n        # If this was a request with a primary key included, then at the\n        # point that we got a good result, we know that we're done and can\n        # return the result.\n        if pk:\n            # Make the results all look the same, for easier parsing\n            # by other methods.\n            #\n            # Note that the `get` method will effectively undo this operation,\n            # but that's a good thing, because we might use `get` without a\n            # primary key.\n            return {'count': 1, 'results': [resp]}\n\n        # Did we get zero results back when we shouldn't?\n        # If so, this is an error, and we need to complain.\n        if fail_on_no_results and resp['count'] == 0:\n            raise exc.NotFound('The requested object could not be found.')\n\n        # Did we get more than one result back?\n        # If so, this is also an error, and we need to complain.\n        if fail_on_multiple_results and resp['count'] >= 2:\n            raise exc.MultipleResults('Expected one result, got %d. Possibly caused by not providing required '\n                                      'fields. Please tighten your criteria.' % resp['count'])\n\n        # Return the response.\n        return resp", "response": "Read an object from the Ansible Tower API."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwrite the object to the resource store.", "response": "def write(self, pk=None, create_on_missing=False, fail_on_found=False, force_on_exists=True, **kwargs):\n        \"\"\"\n        =====API DOCS=====\n        Modify the given object using the Ansible Tower API.\n\n        :param pk: Primary key of the resource to be read. Tower CLI will only attempt to read that object\n                   if ``pk`` is provided (not ``None``).\n        :type pk: int\n        :param create_on_missing: Flag that if set, a new object is created if ``pk`` is not set and objects\n                                  matching the appropriate unique criteria is not found.\n        :type create_on_missing: bool\n        :param fail_on_found: Flag that if set, the operation fails if an object matching the unique criteria\n                              already exists.\n        :type fail_on_found: bool\n        :param force_on_exists: Flag that if set, then if an object is modified based on matching via unique\n                                fields (as opposed to the primary key), other fields are updated based on data\n                                sent; If unset, then the non-unique values are only written in a creation case.\n        :type force_on_exists: bool\n        :param `**kwargs`: Keyword arguments which, all together, will be used as POST/PATCH body to create/modify\n                           the resource object. if ``pk`` is not set, key-value pairs of ``**kwargs`` which are\n                           also in resource's identity will be used to lookup existing reosource.\n        :returns: A dictionary combining the JSON output of the resource, as well as two extra fields: \"changed\",\n                  a flag indicating if the resource is created or successfully updated; \"id\", an integer which\n                  is the primary key of the specified object.\n        :rtype: dict\n        :raises tower_cli.exceptions.BadRequest: When required fields are missing in ``**kwargs`` when creating\n                                                 a new resource object.\n\n        =====API DOCS=====\n        \"\"\"\n        existing_data = {}\n\n        # Remove default values (anything where the value is None).\n        self._pop_none(kwargs)\n\n        # Determine which record we are writing, if we weren't given a primary key.\n        if not pk:\n            debug.log('Checking for an existing record.', header='details')\n            existing_data = self._lookup(\n                fail_on_found=fail_on_found, fail_on_missing=not create_on_missing, include_debug_header=False,\n                **kwargs\n            )\n            if existing_data:\n                pk = existing_data['id']\n        else:\n            # We already know the primary key, but get the existing data.\n            # This allows us to know whether the write made any changes.\n            debug.log('Getting existing record.', header='details')\n            existing_data = self.get(pk)\n\n        # Sanity check: Are we missing required values?\n        # If we don't have a primary key, then all required values must be set, and if they're not, it's an error.\n        missing_fields = []\n        for i in self.fields:\n            if i.key not in kwargs and i.name not in kwargs and i.required:\n                missing_fields.append(i.key or i.name)\n        if missing_fields and not pk:\n            raise exc.BadRequest('Missing required fields: %s' % ', '.join(missing_fields).replace('_', '-'))\n\n        # Sanity check: Do we need to do a write at all?\n        # If `force_on_exists` is False and the record was, in fact, found, then no action is required.\n        if pk and not force_on_exists:\n            debug.log('Record already exists, and --force-on-exists is off; do nothing.', header='decision', nl=2)\n            answer = OrderedDict((('changed', False), ('id', pk)))\n            answer.update(existing_data)\n            return answer\n\n        # Similarly, if all existing data matches our write parameters, there's no need to do anything.\n        if all([kwargs[k] == existing_data.get(k, None) for k in kwargs.keys()]):\n            debug.log('All provided fields match existing data; do nothing.', header='decision', nl=2)\n            answer = OrderedDict((('changed', False), ('id', pk)))\n            answer.update(existing_data)\n            return answer\n\n        # Reinsert None for special case of null association\n        for key in kwargs:\n            if kwargs[key] == 'null':\n                kwargs[key] = None\n\n        # Get the URL and method to use for the write.\n        url = self.endpoint\n        method = 'POST'\n        if pk:\n            url = self._get_patch_url(url, pk)\n            method = 'PATCH'\n\n        # If debugging is on, print the URL and data being sent.\n        debug.log('Writing the record.', header='details')\n\n        # Actually perform the write.\n        r = getattr(client, method.lower())(url, data=kwargs)\n\n        # At this point, we know the write succeeded, and we know that data was changed in the process.\n        answer = OrderedDict((('changed', True), ('id', r.json()['id'])))\n        answer.update(r.json())\n        return answer"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef delete(self, pk=None, fail_on_missing=False, **kwargs):\n        # If we weren't given a primary key, determine which record we're deleting.\n        if not pk:\n            existing_data = self._lookup(fail_on_missing=fail_on_missing, **kwargs)\n            if not existing_data:\n                return {'changed': False}\n            pk = existing_data['id']\n\n        # Attempt to delete the record. If it turns out the record doesn't exist, handle the 404 appropriately\n        # (this is an okay response if `fail_on_missing` is False).\n        url = '%s%s/' % (self.endpoint, pk)\n        debug.log('DELETE %s' % url, fg='blue', bold=True)\n        try:\n            client.delete(url)\n            return {'changed': True}\n        except exc.NotFound:\n            if fail_on_missing:\n                raise\n            return {'changed': False}", "response": "Delete the object with the given primary key."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nretrieve one and exactly one object from the database.", "response": "def get(self, pk=None, **kwargs):\n        \"\"\"Return one and exactly one object.\n\n        Lookups may be through a primary key, specified as a positional argument, and/or through filters specified\n        through keyword arguments.\n\n        If the number of results does not equal one, raise an exception.\n\n        =====API DOCS=====\n        Retrieve one and exactly one object.\n\n        :param pk: Primary key of the resource to be read. Tower CLI will only attempt to read *that* object\n                   if ``pk`` is provided (not ``None``).\n        :type pk: int\n        :param `**kwargs`: Keyword arguments used to look up resource object to retrieve if ``pk`` is not provided.\n        :returns: loaded JSON of the retrieved resource object.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        if kwargs.pop('include_debug_header', True):\n            debug.log('Getting the record.', header='details')\n        response = self.read(pk=pk, fail_on_no_results=True, fail_on_multiple_results=True, **kwargs)\n        return response['results'][0]"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef list(self, all_pages=False, **kwargs):\n        # TODO: Move to a field callback method to make it generic\n        # If multiple statuses where given, add OR queries for each of them\n        if kwargs.get('status', None) and ',' in kwargs['status']:\n            all_status = kwargs.pop('status').strip(',').split(',')\n            queries = list(kwargs.pop('query', ()))\n            for status in all_status:\n                if status in STATUS_CHOICES:\n                    queries.append(('or__status', status))\n                else:\n                    raise exc.TowerCLIError('This status does not exist: {}'.format(status))\n            kwargs['query'] = tuple(queries)\n\n        # If the `all_pages` flag is set, then ignore any page that might also be sent.\n        if all_pages:\n            kwargs.pop('page', None)\n            kwargs.pop('page_size', None)\n\n        # Get the response.\n        debug.log('Getting records.', header='details')\n        response = self.read(**kwargs)\n\n        # Convert next and previous to int\n        self._convert_pagenum(response)\n\n        # If we were asked for all pages, keep retrieving pages until we have them all.\n        if all_pages and response['next']:\n            cursor = copy(response)\n            while cursor['next']:\n                cursor = self.read(**dict(kwargs, page=cursor['next']))\n                self._convert_pagenum(cursor)\n                response['results'] += cursor['results']\n                response['count'] += cursor['count']\n            response['next'] = None\n\n        # Done; return the response\n        return response", "response": "Retrieve a list of objects from the Tower API."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndisassociating the other record from the me record.", "response": "def _disassoc(self, url_fragment, me, other):\n        \"\"\"Disassociate the `other` record from the `me` record.\"\"\"\n\n        # Get the endpoint for foreign records within this object.\n        url = self.endpoint + '%d/%s/' % (me, url_fragment)\n\n        # Attempt to determine whether the other record already is absent, for the \"changed\" moniker.\n        r = client.get(url, params={'id': other}).json()\n        if r['count'] == 0:\n            return {'changed': False}\n\n        # Send a request removing the foreign record from this one.\n        r = client.post(url, data={'disassociate': True, 'id': other})\n        return {'changed': True}"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncopy an object from the API to the new name.", "response": "def copy(self, pk=None, new_name=None, **kwargs):\n        \"\"\"Copy an object.\n\n        Only the ID is used for the lookup. All provided fields are used to override the old data from the\n        copied resource.\n\n        =====API DOCS=====\n        Copy an object.\n\n        :param pk: Primary key of the resource object to be copied\n        :param new_name: The new name to give the resource if deep copying via the API\n        :type pk: int\n        :param `**kwargs`: Keyword arguments of fields whose given value will override the original value.\n        :returns: loaded JSON of the copied new resource object.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        orig = self.read(pk, fail_on_no_results=True, fail_on_multiple_results=True)\n        orig = orig['results'][0]\n        # Remove default values (anything where the value is None).\n        self._pop_none(kwargs)\n\n        newresource = copy(orig)\n        newresource.pop('id')\n        basename = newresource['name'].split('@', 1)[0].strip()\n\n        # Modify data to fit the call pattern of the tower-cli method\n        for field in self.fields:\n            if field.multiple and field.name in newresource:\n                newresource[field.name] = (newresource.get(field.name),)\n\n        if new_name is None:\n            # copy client-side, the old mechanism\n            newresource['name'] = \"%s @ %s\" % (basename, time.strftime('%X'))\n            newresource.update(kwargs)\n\n            return self.write(create_on_missing=True, fail_on_found=True,\n                              **newresource)\n        else:\n            # copy server-side, the new mechanism\n            if kwargs:\n                raise exc.TowerCLIError('Cannot override {} and also use --new-name.'.format(kwargs.keys()))\n            copy_endpoint = '{}/{}/copy/'.format(self.endpoint.strip('/'), pk)\n            return client.post(copy_endpoint, data={'name': new_name}).json()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmodifying an already existing resource object.", "response": "def modify(self, pk=None, create_on_missing=False, **kwargs):\n        \"\"\"Modify an already existing object.\n\n        Fields in the resource's `identity` tuple can be used in lieu of a primary key for a lookup; in such a case,\n        only other fields are written.\n\n        To modify unique fields, you must use the primary key for the lookup.\n\n        =====API DOCS=====\n        Modify an already existing object.\n\n        :param pk: Primary key of the resource to be modified.\n        :type pk: int\n        :param create_on_missing: Flag that if set, a new object is created if ``pk`` is not set and objects\n                                  matching the appropriate unique criteria is not found.\n        :type create_on_missing: bool\n        :param `**kwargs`: Keyword arguments which, all together, will be used as PATCH body to modify the\n                           resource object. if ``pk`` is not set, key-value pairs of ``**kwargs`` which are\n                           also in resource's identity will be used to lookup existing reosource.\n        :returns: A dictionary combining the JSON output of the modified resource, as well as two extra fields:\n                  \"changed\", a flag indicating if the resource is successfully updated; \"id\", an integer which\n                  is the primary key of the updated object.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        return self.write(pk, create_on_missing=create_on_missing, force_on_exists=True, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the data about the last job run off of that UJT.", "response": "def last_job_data(self, pk=None, **kwargs):\n        \"\"\"\n        Internal utility function for Unified Job Templates. Returns data about the last job run off of that UJT\n        \"\"\"\n        ujt = self.get(pk, include_debug_header=True, **kwargs)\n\n        # Determine the appropriate inventory source update.\n        if 'current_update' in ujt['related']:\n            debug.log('A current job; retrieving it.', header='details')\n            return client.get(ujt['related']['current_update'][7:]).json()\n        elif ujt['related'].get('last_update', None):\n            debug.log('No current job or update exists; retrieving the most recent.', header='details')\n            return client.get(ujt['related']['last_update'][7:]).json()\n        else:\n            raise exc.NotFound('No related jobs or updates exist.')"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nprint out the stdout of a unified job.", "response": "def stdout(self, pk, start_line=None, end_line=None, outfile=sys.stdout, **kwargs):\n        \"\"\"\n        Print out the standard out of a unified job to the command line or output file.\n        For Projects, print the standard out of most recent update.\n        For Inventory Sources, print standard out of most recent sync.\n        For Jobs, print the job's standard out.\n        For Workflow Jobs, print a status table of its jobs.\n\n        =====API DOCS=====\n        Print out the standard out of a unified job to the command line or output file.\n        For Projects, print the standard out of most recent update.\n        For Inventory Sources, print standard out of most recent sync.\n        For Jobs, print the job's standard out.\n        For Workflow Jobs, print a status table of its jobs.\n\n        :param pk: Primary key of the job resource object to be monitored.\n        :type pk: int\n        :param start_line: Line at which to start printing job output\n        :param end_line: Line at which to end printing job output\n        :param outfile: Alternative file than stdout to write job stdout to.\n        :type outfile: file\n        :param `**kwargs`: Keyword arguments used to look up job resource object to monitor if ``pk`` is\n                           not provided.\n        :returns: A dictionary containing changed=False\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n\n        # resource is Unified Job Template\n        if self.unified_job_type != self.endpoint:\n            unified_job = self.last_job_data(pk, **kwargs)\n            pk = unified_job['id']\n        # resource is Unified Job, but pk not given\n        elif not pk:\n            unified_job = self.get(**kwargs)\n            pk = unified_job['id']\n\n        content = self.lookup_stdout(pk, start_line, end_line)\n        opened = False\n        if isinstance(outfile, six.string_types):\n            outfile = open(outfile, 'w')\n            opened = True\n        if len(content) > 0:\n            click.echo(content, nl=1, file=outfile)\n        if opened:\n            outfile.close()\n\n        return {\"changed\": False}"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef monitor(self, pk, parent_pk=None, timeout=None, interval=0.5, outfile=sys.stdout, **kwargs):\n\n        # If we do not have the unified job info, infer it from parent\n        if pk is None:\n            pk = self.last_job_data(parent_pk, **kwargs)['id']\n        job_endpoint = '%s%s/' % (self.unified_job_type, pk)\n\n        # Pause until job is in running state\n        self.wait(pk, exit_on=['running', 'successful'], outfile=outfile)\n\n        # Loop initialization\n        start = time.time()\n        start_line = 0\n        result = client.get(job_endpoint).json()\n\n        click.echo('\\033[0;91m------Starting Standard Out Stream------\\033[0m', nl=2, file=outfile)\n\n        # Poll the Ansible Tower instance for status and content, and print standard out to the out file\n        while not result['failed'] and result['status'] != 'successful':\n\n            result = client.get(job_endpoint).json()\n\n            # Put the process to sleep briefly.\n            time.sleep(interval)\n\n            # Make request to get standard out\n            content = self.lookup_stdout(pk, start_line, full=False)\n\n            # In the first moments of running the job, the standard out\n            # may not be available yet\n            if not content.startswith(\"Waiting for results\"):\n                line_count = len(content.splitlines())\n                start_line += line_count\n                click.echo(content, nl=0, file=outfile)\n\n            if timeout and time.time() - start > timeout:\n                raise exc.Timeout('Monitoring aborted due to timeout.')\n\n        # Special final line for closure with workflow jobs\n        if self.endpoint == '/workflow_jobs/':\n            click.echo(self.lookup_stdout(pk, start_line, full=True), nl=1)\n\n        click.echo('\\033[0;91m------End of Standard Out Stream--------\\033[0m', nl=2, file=outfile)\n\n        if result['failed']:\n            raise exc.JobFailure('Job failed.')\n\n        # Return the job ID and other response data\n        answer = OrderedDict((('changed', True), ('id', pk)))\n        answer.update(result)\n        # Make sure to return ID of resource and not update number relevant for project creation and update\n        if parent_pk:\n            answer['id'] = parent_pk\n        else:\n            answer['id'] = pk\n        return answer", "response": "Monitor a unified job resource object."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwaiting for a running job to finish.", "response": "def wait(self, pk, parent_pk=None, min_interval=1, max_interval=30, timeout=None, outfile=sys.stdout,\n             exit_on=['successful'], **kwargs):\n        \"\"\"\n        Wait for a running job to finish. Blocks further input until the job completes (whether successfully\n        or unsuccessfully) and a final status can be given.\n\n        =====API DOCS=====\n        Wait for a job resource object to enter certain status.\n\n        :param pk: Primary key of the job resource object to wait.\n        :type pk: int\n        :param parent_pk: Primary key of the unified job template resource object whose latest job run will be\n                          waited if ``pk`` is not set.\n        :type parent_pk: int\n        :param timeout: Number in seconds after which this method will time out.\n        :type timeout: float\n        :param min_interval: Minimum polling interval to request an update from Tower.\n        :type min_interval: float\n        :param max_interval: Maximum polling interval to request an update from Tower.\n        :type max_interval: float\n        :param outfile: Alternative file than stdout to write job status updates on.\n        :type outfile: file\n        :param exit_on: Job resource object statuses to wait on.\n        :type exit_on: array\n        :param `**kwargs`: Keyword arguments used to look up job resource object to wait if ``pk`` is\n                           not provided.\n        :returns: A dictionary combining the JSON output of the status-changed job resource object, as well\n                  as two extra fields: \"changed\", a flag indicating if the job resource object is status-changed\n                  as expected; \"id\", an integer which is the primary key of the job resource object being\n                  status-changed.\n        :rtype: dict\n        :raises tower_cli.exceptions.Timeout: When wait time reaches time out.\n        :raises tower_cli.exceptions.JobFailure: When the job being waited on runs into failure.\n        =====API DOCS=====\n        \"\"\"\n        # If we do not have the unified job info, infer it from parent\n        if pk is None:\n            pk = self.last_job_data(parent_pk, **kwargs)['id']\n        job_endpoint = '%s%s/' % (self.unified_job_type, pk)\n\n        dots = itertools.cycle([0, 1, 2, 3])\n        longest_string = 0\n        interval = min_interval\n        start = time.time()\n\n        # Poll the Ansible Tower instance for status, and print the status to the outfile (usually standard out).\n        #\n        # Note that this is one of the few places where we use `secho` even though we're in a function that might\n        # theoretically be imported and run in Python.  This seems fine; outfile can be set to /dev/null and very\n        # much the normal use for this method should be CLI monitoring.\n        result = client.get(job_endpoint).json()\n        last_poll = time.time()\n        timeout_check = 0\n        while result['status'] not in exit_on:\n            # If the job has failed, we want to raise an Exception for that so we get a non-zero response.\n            if result['failed']:\n                if is_tty(outfile) and not settings.verbose:\n                    secho('\\r' + ' ' * longest_string + '\\n', file=outfile)\n                raise exc.JobFailure('Job failed.')\n\n            # Sanity check: Have we officially timed out?\n            # The timeout check is incremented below, so this is checking to see if we were timed out as of\n            # the previous iteration. If we are timed out, abort.\n            if timeout and timeout_check - start > timeout:\n                raise exc.Timeout('Monitoring aborted due to timeout.')\n\n            # If the outfile is a TTY, print the current status.\n            output = '\\rCurrent status: %s%s' % (result['status'], '.' * next(dots))\n            if longest_string > len(output):\n                output += ' ' * (longest_string - len(output))\n            else:\n                longest_string = len(output)\n            if is_tty(outfile) and not settings.verbose:\n                secho(output, nl=False, file=outfile)\n\n            # Put the process to sleep briefly.\n            time.sleep(0.2)\n\n            # Sanity check: Have we reached our timeout?\n            # If we're about to time out, then we need to ensure that we do one last check.\n            #\n            # Note that the actual timeout will be performed at the start of the **next** iteration,\n            # so there's a chance for the job's completion to be noted first.\n            timeout_check = time.time()\n            if timeout and timeout_check - start > timeout:\n                last_poll -= interval\n\n            # If enough time has elapsed, ask the server for a new status.\n            #\n            # Note that this doesn't actually do a status check every single time; we want the \"spinner\" to\n            # spin even if we're not actively doing a check.\n            #\n            # So, what happens is that we are \"counting down\" (actually up) to the next time that we intend\n            # to do a check, and once that time hits, we do the status check as part of the normal cycle.\n            if time.time() - last_poll > interval:\n                result = client.get(job_endpoint).json()\n                last_poll = time.time()\n                interval = min(interval * 1.5, max_interval)\n\n                # If the outfile is *not* a TTY, print a status update when and only when we make an actual\n                # check to job status.\n                if not is_tty(outfile) or settings.verbose:\n                    click.echo('Current status: %s' % result['status'], file=outfile)\n\n            # Wipe out the previous output\n            if is_tty(outfile) and not settings.verbose:\n                secho('\\r' + ' ' * longest_string, file=outfile, nl=False)\n                secho('\\r', file=outfile, nl=False)\n\n        # Return the job ID and other response data\n        answer = OrderedDict((('changed', True), ('id', pk)))\n        answer.update(result)\n        # Make sure to return ID of resource and not update number relevant for project creation and update\n        if parent_pk:\n            answer['id'] = parent_pk\n        else:\n            answer['id'] = pk\n        return answer"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef status(self, pk=None, detail=False, **kwargs):\n        # Remove default values (anything where the value is None).\n        self._pop_none(kwargs)\n\n        # Search for the record if pk not given\n        if not pk:\n            job = self.get(include_debug_header=True, **kwargs)\n        # Get the job from Ansible Tower if pk given\n        else:\n            debug.log('Asking for job status.', header='details')\n            finished_endpoint = '%s%s/' % (self.endpoint, pk)\n            job = client.get(finished_endpoint).json()\n\n        # In most cases, we probably only want to know the status of the job and the amount of time elapsed.\n        # However, if we were asked for verbose information, provide it.\n        if detail:\n            return job\n\n        # Print just the information we need.\n        return {\n            'elapsed': job['elapsed'],\n            'failed': job['failed'],\n            'status': job['status'],\n        }", "response": "Print the current status of the unified job."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef cancel(self, pk=None, fail_if_not_running=False, **kwargs):\n        # Search for the record if pk not given\n        if not pk:\n            existing_data = self.get(**kwargs)\n            pk = existing_data['id']\n\n        cancel_endpoint = '%s%s/cancel/' % (self.endpoint, pk)\n        # Attempt to cancel the job.\n        try:\n            client.post(cancel_endpoint)\n            changed = True\n        except exc.MethodNotAllowed:\n            changed = False\n            if fail_if_not_running:\n                raise exc.TowerCLIError('Job not running.')\n\n        # Return a success.\n        return {'status': 'canceled', 'changed': changed}", "response": "Cancels a currently running job."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the survey specification for a job template.", "response": "def survey(self, pk=None, **kwargs):\n        \"\"\"Get the survey_spec for the job template.\n        To write a survey, use the modify command with the --survey-spec parameter.\n\n        =====API DOCS=====\n        Get the survey specification of a resource object.\n\n        :param pk: Primary key of the resource to retrieve survey from. Tower CLI will only attempt to\n                   read *that* object if ``pk`` is provided (not ``None``).\n        :type pk: int\n        :param `**kwargs`: Keyword arguments used to look up resource object to retrieve survey if ``pk``\n                           is not provided.\n        :returns: loaded JSON of the retrieved survey specification of the resource object.\n        :rtype: dict\n        =====API DOCS=====\n        \"\"\"\n        job_template = self.get(pk=pk, **kwargs)\n        if settings.format == 'human':\n            settings.format = 'json'\n        return client.get(self._survey_endpoint(job_template['id'])).json()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nbatching update all related inventory sources of the given inventory.", "response": "def batch_update(self, pk=None, **kwargs):\n        \"\"\"Update all related inventory sources of the given inventory.\n\n        Note global option --format is not available here, as the output would always be JSON-formatted.\n\n        =====API DOCS=====\n        Update all related inventory sources of the given inventory.\n\n        :param pk: Primary key of the given inventory.\n        :type pk: int\n        :param `**kwargs`: Keyword arguments list of available fields used for searching resource objects.\n        :returns: A JSON object of update status of the given inventory.\n        :rtype: dict\n        =====API DOCS=====\n        \"\"\"\n        res = self.get(pk=pk, **kwargs)\n        url = self.endpoint + '%d/%s/' % (res['id'], 'update_inventory_sources')\n        return client.post(url, data={}).json()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\npromoting the actor field to the top - level field as", "response": "def read(self, *args, **kwargs):\n        '''\n        Do extra processing so we can display the actor field as\n        a top-level field\n        '''\n        if 'actor' in kwargs:\n            kwargs['actor'] = kwargs.pop('actor')\n        r = super(Resource, self).read(*args, **kwargs)\n        if 'results' in r:\n            for d in r['results']:\n                self._promote_actor(d)\n        else:\n            self._promote_actor(d)\n        return r"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nlogging the given output to stderr if and only if we are in verbose mode.", "response": "def log(s, header='', file=sys.stderr, nl=1, **kwargs):\n    \"\"\"Log the given output to stderr if and only if we are in\n    verbose mode.\n\n    If we are not in verbose mode, this is a no-op.\n    \"\"\"\n    # Sanity check: If we are not in verbose mode, this is a no-op.\n    if not settings.verbose:\n        return\n\n    # Construct multi-line string to stderr if header is provided.\n    if header:\n        word_arr = s.split(' ')\n        multi = []\n        word_arr.insert(0, '%s:' % header.upper())\n        i = 0\n        while i < len(word_arr):\n            to_add = ['***']\n            count = 3\n            while count <= 79:\n                count += len(word_arr[i]) + 1\n                if count <= 79:\n                    to_add.append(word_arr[i])\n                    i += 1\n                    if i == len(word_arr):\n                        break\n            # Handle corner case of extra-long word longer than 75 characters.\n            if len(to_add) == 1:\n                to_add.append(word_arr[i])\n                i += 1\n            if i != len(word_arr):\n                count -= len(word_arr[i]) + 1\n            to_add.append('*' * (78 - count))\n            multi.append(' '.join(to_add))\n        s = '\\n'.join(multi)\n        lines = len(multi)\n    else:\n        lines = 1\n\n    # If `nl` is an int greater than the number of rows of a message,\n    # add the appropriate newlines to the output.\n    if isinstance(nl, int) and nl > lines:\n        s += '\\n' * (nl - lines)\n\n    # Output to stderr.\n    return secho(s, file=file, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef configure_model(self, attrs, field_name):\n        '''\n        Hook for ResourceMeta class to call when initializing model class.\n        Saves fields obtained from resource class backlinks\n        '''\n        self.relationship = field_name\n        self._set_method_names(relationship=field_name)\n        if self.res_name is None:\n            self.res_name = grammar.singularize(attrs.get('endpoint', 'unknown').strip('/'))", "response": "Hook for ResourceMeta class to call when initializing model class."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _produce_raw_method(self):\n        '''\n        Returns a callable which becomes the associate or disassociate\n        method for the related field.\n        Method can be overridden to add additional functionality, but\n        `_produce_method` may also need to be subclassed to decorate\n        it appropriately.\n        '''\n\n        def method(res_self, **kwargs):\n            obj_pk = kwargs.get(method._res_name)\n            other_obj_pk = kwargs.get(method._other_name)\n            internal_method = getattr(res_self, method._internal_name)\n            return internal_method(method._relationship, obj_pk, other_obj_pk)\n\n        return method", "response": "Returns a callable which becomes the associate or disassociate\n        method for the related field."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncreating a new label under the specified job template.", "response": "def create(self, fail_on_found=False, force_on_exists=False, **kwargs):\n        \"\"\"Create a new label.\n\n        There are two types of label creation: isolatedly creating a new label and creating a new label under\n        a job template. Here the two types are discriminated by whether to provide --job-template option.\n\n        Fields in the resource's `identity` tuple are used for a lookup; if a match is found, then no-op (unless\n        `force_on_exists` is set) but do not fail (unless `fail_on_found` is set).\n\n        =====API DOCS=====\n        Create a label.\n\n        :param job_template: Primary key or name of the job template for the created label to associate to.\n        :type job_template: str\n        :param fail_on_found: Flag that if set, the operation fails if an object matching the unique criteria\n                              already exists.\n        :type fail_on_found: bool\n        :param force_on_exists: Flag that if set, then if a match is found on unique fields, other fields will\n                                be updated to the provided values.; If unset, a match causes the request to be\n                                a no-op.\n        :type force_on_exists: bool\n        :param `**kwargs`: Keyword arguments which, all together, will be used as POST body to create the\n                           resource object.\n        :returns: A dictionary combining the JSON output of the created resource, as well as two extra fields:\n                  \"changed\", a flag indicating if the resource is created successfully; \"id\", an integer which\n                  is the primary key of the created object.\n        :rtype: dict\n        :raises tower_cli.exceptions.TowerCLIError: When the label already exists and ``fail_on_found`` flag is on.\n\n        =====API DOCS=====\n        \"\"\"\n        jt_id = kwargs.pop('job_template', None)\n        old_endpoint = self.endpoint\n        if jt_id is not None:\n            jt = get_resource('job_template')\n            jt.get(pk=jt_id)\n            try:\n                label_id = self.get(name=kwargs.get('name', None), organization=kwargs.get('organization', None))['id']\n            except exc.NotFound:\n                pass\n            else:\n                if fail_on_found:\n                    raise exc.TowerCLIError('Label already exists and fail-on-found is switched on. Please use'\n                                            ' \"associate_label\" method of job_template instead.')\n                else:\n                    debug.log('Label already exists, associating with job template.', header='details')\n                    return jt.associate_label(job_template=jt_id, label=label_id)\n            self.endpoint = '/job_templates/%d/labels/' % jt_id\n        result = super(Resource, self).create(fail_on_found=fail_on_found, force_on_exists=force_on_exists, **kwargs)\n        self.endpoint = old_endpoint\n        return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef version():\n\n    # Print out the current version of Tower CLI.\n    click.echo('Tower CLI %s' % __version__)\n\n    # Print out the current API version of the current code base.\n    click.echo('API %s' % CUR_API_VERSION)\n\n    # Attempt to connect to the Ansible Tower server.\n    # If we succeed, print a version; if not, generate a failure.\n    try:\n        r = client.get('/config/')\n    except RequestException as ex:\n        raise exc.TowerCLIError('Could not connect to Ansible Tower.\\n%s' %\n                                six.text_type(ex))\n    config = r.json()\n    license = config.get('license_info', {}).get('license_type', 'open')\n    if license == 'open':\n        server_type = 'AWX'\n    else:\n        server_type = 'Ansible Tower'\n    click.echo('%s %s' % (server_type, config['version']))\n\n    # Print out Ansible version of server\n    click.echo('Ansible %s' % config['ansible_version'])", "response": "Display full version information."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _echo_setting(key):\n    value = getattr(settings, key)\n    secho('%s: ' % key, fg='magenta', bold=True, nl=False)\n    secho(\n        six.text_type(value),\n        bold=True,\n        fg='white' if isinstance(value, six.text_type) else 'cyan',\n    )", "response": "Echo a setting to the CLI."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading or write a configuration file for the given key and value.", "response": "def config(key=None, value=None, scope='user', global_=False, unset=False):\n    \"\"\"Read or write tower-cli configuration.\n\n    `tower config` saves the given setting to the appropriate Tower CLI;\n    either the user's ~/.tower_cli.cfg file, or the /etc/tower/tower_cli.cfg\n    file if --global is used.\n\n    Writing to /etc/tower/tower_cli.cfg is likely to require heightened\n    permissions (in other words, sudo).\n    \"\"\"\n    # If the old-style `global_` option is set, issue a deprecation notice.\n    if global_:\n        scope = 'global'\n        warnings.warn('The `--global` option is deprecated and will be '\n                      'removed. Use `--scope=global` to get the same effect.',\n                      DeprecationWarning)\n\n    # If no key was provided, print out the current configuration\n    # in play.\n    if not key:\n        seen = set()\n        parser_desc = {\n            'runtime': 'Runtime options.',\n            'environment': 'Options from environment variables.',\n            'local': 'Local options (set with `tower-cli config '\n                     '--scope=local`; stored in .tower_cli.cfg of this '\n                     'directory or a parent)',\n            'user': 'User options (set with `tower-cli config`; stored in '\n                    '~/.tower_cli.cfg).',\n            'global': 'Global options (set with `tower-cli config '\n                      '--scope=global`, stored in /etc/tower/tower_cli.cfg).',\n            'defaults': 'Defaults.',\n        }\n\n        # Iterate over each parser (English: location we can get settings from)\n        # and print any settings that we haven't already seen.\n        #\n        # We iterate over settings from highest precedence to lowest, so any\n        # seen settings are overridden by the version we iterated over already.\n        click.echo('')\n        for name, parser in zip(settings._parser_names, settings._parsers):\n            # Determine if we're going to see any options in this\n            # parser that get echoed.\n            will_echo = False\n            for option in parser.options('general'):\n                if option in seen:\n                    continue\n                will_echo = True\n\n            # Print a segment header\n            if will_echo:\n                secho('# %s' % parser_desc[name], fg='green', bold=True)\n\n            # Iterate over each option in the parser and, if we haven't\n            # already seen an option at higher precedence, print it.\n            for option in parser.options('general'):\n                if option in seen:\n                    continue\n                _echo_setting(option)\n                seen.add(option)\n\n            # Print a nice newline, for formatting.\n            if will_echo:\n                click.echo('')\n        return\n\n    # Sanity check: Is this a valid configuration option? If it's not\n    # a key we recognize, abort.\n    if not hasattr(settings, key):\n        raise exc.TowerCLIError('Invalid configuration option \"%s\".' % key)\n\n    # Sanity check: The combination of a value and --unset makes no\n    # sense.\n    if value and unset:\n        raise exc.UsageError('Cannot provide both a value and --unset.')\n\n    # If a key was provided but no value was provided, then just\n    # print the current value for that key.\n    if key and not value and not unset:\n        _echo_setting(key)\n        return\n\n    # Okay, so we're *writing* a key. Let's do this.\n    # First, we need the appropriate file.\n    filename = os.path.expanduser('~/.tower_cli.cfg')\n    if scope == 'global':\n        if not os.path.isdir('/etc/tower/'):\n            raise exc.TowerCLIError('/etc/tower/ does not exist, and this '\n                                    'command cowardly declines to create it.')\n        filename = '/etc/tower/tower_cli.cfg'\n    elif scope == 'local':\n        filename = '.tower_cli.cfg'\n\n    # Read in the appropriate config file, write this value, and save\n    # the result back to the file.\n    parser = Parser()\n    parser.add_section('general')\n    parser.read(filename)\n    if unset:\n        parser.remove_option('general', key)\n    else:\n        parser.set('general', key, value)\n    with open(filename, 'w') as config_file:\n        parser.write(config_file)\n\n    # Give rw permissions to user only fix for issue number 48\n    try:\n        os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR)\n    except Exception as e:\n        warnings.warn(\n            'Unable to set permissions on {0} - {1} '.format(filename, e),\n            UserWarning\n            )\n    click.echo('Configuration updated successfully.')"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef login(username, password, scope, client_id, client_secret, verbose):\n    if not supports_oauth():\n        raise exc.TowerCLIError(\n            'This version of Tower does not support OAuth2.0. Set credentials using tower-cli config.'\n        )\n\n    # Explicitly set a basic auth header for PAT acquisition (so that we don't\n    # try to auth w/ an existing user+pass or oauth2 token in a config file)\n\n    req = collections.namedtuple('req', 'headers')({})\n    if client_id and client_secret:\n        HTTPBasicAuth(client_id, client_secret)(req)\n        req.headers['Content-Type'] = 'application/x-www-form-urlencoded'\n        r = client.post(\n            '/o/token/',\n            data={\n                \"grant_type\": \"password\",\n                \"username\": username,\n                \"password\": password,\n                \"scope\": scope\n            },\n            headers=req.headers\n        )\n    elif client_id:\n        req.headers['Content-Type'] = 'application/x-www-form-urlencoded'\n        r = client.post(\n            '/o/token/',\n            data={\n                \"grant_type\": \"password\",\n                \"username\": username,\n                \"password\": password,\n                \"client_id\": client_id,\n                \"scope\": scope\n            },\n            headers=req.headers\n        )\n    else:\n        HTTPBasicAuth(username, password)(req)\n        r = client.post(\n            '/users/{}/personal_tokens/'.format(username),\n            data={\"description\": \"Tower CLI\", \"application\": None, \"scope\": scope},\n            headers=req.headers\n        )\n\n    if r.ok:\n        result = r.json()\n        result.pop('summary_fields', None)\n        result.pop('related', None)\n        if client_id:\n            token = result.pop('access_token', None)\n        else:\n            token = result.pop('token', None)\n        if settings.verbose:\n            # only print the actual token if -v\n            result['token'] = token\n        secho(json.dumps(result, indent=1), fg='blue', bold=True)\n        config.main(['oauth_token', token, '--scope=user'])", "response": "Logs in a user with a new OAuth2 token."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nexports assets from Tower.", "response": "def receive(organization=None, user=None, team=None, credential_type=None, credential=None,\n            notification_template=None, inventory_script=None, inventory=None, project=None, job_template=None,\n            workflow=None, all=None):\n    \"\"\"Export assets from Tower.\n\n    'tower receive' exports one or more assets from a Tower instance\n\n    For all of the possible assets types the TEXT can either be the assets name\n    (or username for the case of a user) or the keyword all. Specifying all\n    will export all of the assets of that type.\n\n    \"\"\"\n\n    from tower_cli.cli.transfer.receive import Receiver\n    receiver = Receiver()\n    assets_to_export = {}\n    for asset_type in SEND_ORDER:\n        assets_to_export[asset_type] = locals()[asset_type]\n    receiver.receive(all=all, asset_input=assets_to_export)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nimports assets into Tower.", "response": "def send(source=None, prevent=None, exclude=None, secret_management='default', no_color=False):\n    \"\"\"Import assets into Tower.\n\n    'tower send' imports one or more assets into a Tower instance\n\n    The import can take either JSON or YAML.\n    Data can be sent on stdin (i.e. from tower-cli receive pipe) and/or from files\n    or directories passed as parameters.\n\n    If a directory is specified only files that end in .json, .yaml or .yml will be\n    imported. Other files will be ignored.\n    \"\"\"\n\n    from tower_cli.cli.transfer.send import Sender\n    sender = Sender(no_color)\n    sender.send(source, prevent, exclude, secret_management)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef empty(organization=None, user=None, team=None, credential_type=None, credential=None, notification_template=None,\n          inventory_script=None, inventory=None, project=None, job_template=None, workflow=None,\n          all=None, no_color=False):\n    \"\"\"Empties assets from Tower.\n\n    'tower empty' removes all assets from Tower\n\n    \"\"\"\n\n    # Create an import/export object\n    from tower_cli.cli.transfer.cleaner import Cleaner\n    destroyer = Cleaner(no_color)\n    assets_to_export = {}\n    for asset_type in SEND_ORDER:\n        assets_to_export[asset_type] = locals()[asset_type]\n    destroyer.go_ham(all=all, asset_input=assets_to_export)", "response": "Empty the assets from Tower."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nparse out a list of requirements from the given requirements file.", "response": "def parse_requirements(filename):\n    \"\"\"Parse out a list of requirements from the given requirements\n    requirements file.\n    \"\"\"\n    reqs = []\n    version_spec_in_play = None\n\n    # Iterate over each line in the requirements file.\n    for line in open(filename, 'r').read().strip().split('\\n'):\n        # Sanity check: Is this an empty line?\n        # If so, do nothing.\n        if not line.strip():\n            continue\n\n        # If this is just a plain requirement (not a comment), then\n        # add it to the requirements list.\n        if not line.startswith('#'):\n            reqs.append(line)\n            continue\n\n        # \"Header\" comments take the form of \"=== Python {op} {version} ===\",\n        # and make the requirement only matter for those versions.\n        # If this line is a header comment, parse it.\n        match = re.search(r'^# === [Pp]ython (?P<op>[<>=]{1,2}) '\n                          r'(?P<major>[\\d])\\.(?P<minor>[\\d]+) ===[\\s]*$', line)\n        if match:\n            version_spec_in_play = match.groupdict()\n            for key in ('major', 'minor'):\n                version_spec_in_play[key] = int(version_spec_in_play[key])\n            continue\n\n        # If this is a comment that otherwise looks like a package, then it\n        # should be a package applying only to the current version spec.\n        #\n        # We can identify something that looks like a package by a lack\n        # of any spaces.\n        if ' ' not in line[1:].strip() and version_spec_in_play:\n            package = line[1:].strip()\n\n            # Sanity check: Is our version of Python one of the ones currently\n            # in play?\n            op = version_spec_in_play['op']\n            vspec = (version_spec_in_play['major'],\n                     version_spec_in_play['minor'])\n            if '=' in op and sys.version_info[0:2] == vspec:\n                reqs.append(package)\n            elif '>' in op and sys.version_info[0:2] > vspec:\n                reqs.append(package)\n            elif '<' in op and sys.version_info[0:2] < vspec:\n                reqs.append(package)\n\n    # Okay, we should have an entire list of requirements now.\n    return reqs"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef combine_files(*args):\n    file_contents = []\n    for filename in args:\n        with codecs.open(filename, mode='r', encoding='utf8') as f:\n            file_contents.append(f.read())\n    return \"\\n\\n\".join(file_contents)", "response": "returns a string of all the strings in *args combined together with two line breaks between them"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef create(self, organization=None, monitor=False, wait=False,\n               timeout=None, fail_on_found=False, force_on_exists=False,\n               **kwargs):\n        \"\"\"Create a new item of resource, with or w/o org.\n        This would be a shared class with user, but it needs the ability\n        to monitor if the flag is set.\n\n        =====API DOCS=====\n        Create a project and, if related flags are set, monitor or wait the triggered initial project update.\n\n        :param monitor: Flag that if set, immediately calls ``monitor`` on the newly triggered project update\n                        rather than exiting with a success.\n        :type monitor: bool\n        :param wait: Flag that if set, monitor the status of the triggered project update, but do not print\n                     while it is in progress.\n        :type wait: bool\n        :param timeout: If provided with ``monitor`` flag set, this attempt will time out after the given number\n                        of seconds.\n        :type timeout: bool\n        :param fail_on_found: Flag that if set, the operation fails if an object matching the unique criteria\n                              already exists.\n        :type fail_on_found: bool\n        :param force_on_exists: Flag that if set, then if a match is found on unique fields, other fields will\n                                be updated to the provided values.; If unset, a match causes the request to be\n                                a no-op.\n        :type force_on_exists: bool\n        :param `**kwargs`: Keyword arguments which, all together, will be used as POST body to create the\n                           resource object.\n        :returns: A dictionary combining the JSON output of the created resource, as well as two extra fields:\n                  \"changed\", a flag indicating if the resource is created successfully; \"id\", an integer which\n                  is the primary key of the created object.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        if 'job_timeout' in kwargs and 'timeout' not in kwargs:\n            kwargs['timeout'] = kwargs.pop('job_timeout')\n\n        post_associate = False\n        if organization:\n            # Processing the organization flag depends on version\n            debug.log('Checking Organization Relationship.', header='details')\n            r = client.options('/projects/')\n            if 'organization' in r.json().get('actions', {}).get('POST', {}):\n                kwargs['organization'] = organization\n            else:\n                post_associate = True\n\n        # First, run the create method, ignoring the organization given\n        answer = super(Resource, self).write(\n            create_on_missing=True,\n            fail_on_found=fail_on_found, force_on_exists=force_on_exists,\n            **kwargs\n        )\n        project_id = answer['id']\n\n        # If an organization is given, associate it here\n        if post_associate:\n\n            # Get the organization from Tower, will lookup name if needed\n            org_resource = get_resource('organization')\n            org_data = org_resource.get(organization)\n            org_pk = org_data['id']\n\n            debug.log(\"associating the project with its organization\",\n                      header='details', nl=1)\n            org_resource._assoc('projects', org_pk, project_id)\n\n        # if the monitor flag is set, wait for the SCM to update\n        if monitor and answer.get('changed', False):\n            return self.monitor(pk=None, parent_pk=project_id, timeout=timeout)\n        elif wait and answer.get('changed', False):\n            return self.wait(pk=None, parent_pk=project_id, timeout=timeout)\n\n        return answer", "response": "Create a new item of resource with or w/o org."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nmodifies an existing resource.", "response": "def modify(self, pk=None, create_on_missing=False, **kwargs):\n        \"\"\"Modify an already existing.\n\n        To edit the project's organizations, see help for organizations.\n\n        Fields in the resource's `identity` tuple can be used in lieu of a\n        primary key for a lookup; in such a case, only other fields are\n        written.\n\n        To modify unique fields, you must use the primary key for the lookup.\n\n        =====API DOCS=====\n        Modify an already existing project.\n\n        :param pk: Primary key of the resource to be modified.\n        :type pk: int\n        :param create_on_missing: Flag that if set, a new object is created if ``pk`` is not set and objects\n                                  matching the appropriate unique criteria is not found.\n        :type create_on_missing: bool\n        :param `**kwargs`: Keyword arguments which, all together, will be used as PATCH body to modify the\n                           resource object. if ``pk`` is not set, key-value pairs of ``**kwargs`` which are\n                           also in resource's identity will be used to lookup existing reosource.\n        :returns: A dictionary combining the JSON output of the modified resource, as well as two extra fields:\n                  \"changed\", a flag indicating if the resource is successfully updated; \"id\", an integer which\n                  is the primary key of the updated object.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        # Associated with issue #52, the organization can't be modified\n        #    with the 'modify' command. This would create confusion about\n        #    whether its flag is an identifier versus a field to modify.\n        if 'job_timeout' in kwargs and 'timeout' not in kwargs:\n            kwargs['timeout'] = kwargs.pop('job_timeout')\n        return super(Resource, self).write(\n            pk, create_on_missing=create_on_missing,\n            force_on_exists=True, **kwargs\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef update(self, pk=None, create_on_missing=False, monitor=False,\n               wait=False, timeout=None, name=None, organization=None):\n        \"\"\"Trigger a project update job within Ansible Tower.\n        Only meaningful on non-manual projects.\n\n        =====API DOCS=====\n        Update the given project.\n\n        :param pk: Primary key of the project to be updated.\n        :type pk: int\n        :param monitor: Flag that if set, immediately calls ``monitor`` on the newly launched project update\n                        rather than exiting with a success.\n        :type monitor: bool\n        :param wait: Flag that if set, monitor the status of the project update, but do not print while it is\n                     in progress.\n        :type wait: bool\n        :param timeout: If provided with ``monitor`` flag set, this attempt will time out after the given number\n                        of seconds.\n        :type timeout: int\n        :param name: Name of the project to be updated if ``pk`` is not set.\n        :type name: str\n        :param organization: Primary key or name of the organization the project to be updated belonging to if\n                             ``pk`` is not set.\n        :type organization: str\n        :returns: Result of subsequent ``monitor`` call if ``monitor`` flag is on; Result of subsequent ``wait``\n                  call if ``wait`` flag is on; dictionary of \"status\" if none of the two flags are on.\n        :rtype: dict\n        :raises tower_cli.exceptions.CannotStartJob: When the project cannot be updated.\n\n        =====API DOCS=====\n        \"\"\"\n        # First, get the appropriate project.\n        # This should be uniquely identified at this point, and if not, then\n        # we just want the error that `get` will throw to bubble up.\n        project = self.get(pk, name=name, organization=organization)\n        pk = project['id']\n\n        # Determine whether this project is able to be updated.\n        debug.log('Asking whether the project can be updated.',\n                  header='details')\n        result = client.get('/projects/%d/update/' % pk)\n        if not result.json()['can_update']:\n            raise exc.CannotStartJob('Cannot update project.')\n\n        # Okay, this project can be updated, according to Tower.\n        # Commence the update.\n        debug.log('Updating the project.', header='details')\n        result = client.post('/projects/%d/update/' % pk)\n\n        project_update_id = result.json()['project_update']\n\n        # If we were told to monitor the project update's status, do so.\n        if monitor:\n            return self.monitor(project_update_id, parent_pk=pk,\n                                timeout=timeout)\n        elif wait:\n            return self.wait(project_update_id, parent_pk=pk, timeout=timeout)\n\n        # Return the project update ID.\n        return {\n            'id': project_update_id,\n            'changed': True,\n        }", "response": "Update the project with the given primary key."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nprints the status of the specified unified job.", "response": "def status(self, pk=None, detail=False, **kwargs):\n        \"\"\"Print the status of the most recent update.\n\n        =====API DOCS=====\n        Print the status of the most recent update.\n\n        :param pk: Primary key of the resource to retrieve status from.\n        :type pk: int\n        :param detail: Flag that if set, return the full JSON of the job resource rather than a status summary.\n        :type detail: bool\n        :param `**kwargs`: Keyword arguments used to look up resource object to retrieve status from if ``pk``\n                           is not provided.\n        :returns: full loaded JSON of the specified unified job if ``detail`` flag is on; trimed JSON containing\n                  only \"elapsed\", \"failed\" and \"status\" fields of the unified job if ``detail`` flag is off.\n        :rtype: dict\n        =====API DOCS=====\n        \"\"\"\n        # Obtain the most recent project update\n        job = self.last_job_data(pk, **kwargs)\n\n        # In most cases, we probably only want to know the status of the job\n        # and the amount of time elapsed. However, if we were asked for\n        # verbose information, provide it.\n        if detail:\n            return job\n\n        # Print just the information we need.\n        return {\n            'elapsed': job['elapsed'],\n            'failed': job['failed'],\n            'status': job['status'],\n        }"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef convert(self, value, param, ctx):\n        # Protect against corner cases of invalid inputs\n        if not isinstance(value, str):\n            return value\n        if isinstance(value, six.binary_type):\n            value = value.decode('UTF-8')\n        # Read from a file under these cases\n        if value.startswith('@'):\n            filename = os.path.expanduser(value[1:])\n            file_obj = super(Variables, self).convert(filename, param, ctx)\n            if hasattr(file_obj, 'read'):\n                # Sometimes click.File may return a buffer and not a string\n                return file_obj.read()\n            return file_obj\n\n        # No file, use given string\n        return value", "response": "Return file content if file else return value as - is\n       "}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef convert(self, value, param, ctx):\n        choice = super(MappedChoice, self).convert(value, param, ctx)\n        ix = self.choices.index(choice)\n        return self.actual_choices[ix]", "response": "Match against the appropriate choice value using the superclass\n        implementation then return the actual choice."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nconvert the value of a parameter into a value that can be used to create a new entry in the database.", "response": "def convert(self, value, param, ctx):\n        \"\"\"Return the appropriate integer value. If a non-integer is\n        provided, attempt a name-based lookup and return the primary key.\n        \"\"\"\n        resource = tower_cli.get_resource(self.resource_name)\n\n        # Ensure that None is passed through without trying to\n        # do anything.\n        if value is None:\n            return None\n\n        # If we were already given an integer, do nothing.\n        # This ensures that the convert method is idempotent.\n        if isinstance(value, int):\n            return value\n\n        # Do we have a string that contains only digits?\n        # If so, then convert it to an integer and return it.\n        if re.match(r'^[\\d]+$', value):\n            return int(value)\n\n        # Special case to allow disassociations\n        if value == 'null':\n            return value\n\n        # Okay, we have a string. Try to do a name-based lookup on the\n        # resource, and return back the ID that we get from that.\n        #\n        # This has the chance of erroring out, which is fine.\n        try:\n            debug.log('The %s field is given as a name; '\n                      'looking it up.' % param.name, header='details')\n            lookup_data = {resource.identity[-1]: value}\n            rel = resource.get(**lookup_data)\n        except exc.MultipleResults:\n            raise exc.MultipleRelatedError(\n                'Cannot look up {0} exclusively by name, because multiple {0} '\n                'objects exist with that name.\\n'\n                'Please send an ID. You can get the ID for the {0} you want '\n                'with:\\n'\n                '  tower-cli {0} list --name \"{1}\"'.format(self.resource_name,\n                                                           value),\n            )\n        except exc.TowerCLIError as ex:\n            raise exc.RelatedError('Could not get %s. %s' %\n                                   (self.resource_name, str(ex)))\n\n        # Done! Return the ID.\n        return rel['id']"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning filtering parameters to limit a search to the children of a particular node by a particular relationship.", "response": "def _parent_filter(self, parent, relationship, **kwargs):\n        \"\"\"\n        Returns filtering parameters to limit a search to the children\n        of a particular node by a particular relationship.\n        \"\"\"\n        if parent is None or relationship is None:\n            return {}\n        parent_filter_kwargs = {}\n        query_params = ((self._reverse_rel_name(relationship), parent),)\n        parent_filter_kwargs['query'] = query_params\n        if kwargs.get('workflow_job_template', None) is None:\n            parent_data = self.read(pk=parent)['results'][0]\n            parent_filter_kwargs['workflow_job_template'] = parent_data[\n                'workflow_job_template']\n        return parent_filter_kwargs"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nassociating a success node with a parent node.", "response": "def associate_success_node(self, parent, child=None, **kwargs):\n        \"\"\"Add a node to run on success.\n\n        =====API DOCS=====\n        Add a node to run on success.\n\n        :param parent: Primary key of parent node to associate success node to.\n        :type parent: int\n        :param child: Primary key of child node to be associated.\n        :type child: int\n        :param `**kwargs`: Fields used to create child node if ``child`` is not provided.\n        :returns: Dictionary of only one key \"changed\", which indicates whether the association succeeded.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        return self._assoc_or_create('success', parent, child, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndisassociate a success node from a parent node.", "response": "def disassociate_success_node(self, parent, child):\n        \"\"\"Remove success node.\n        The resulatant 2 nodes will both become root nodes.\n\n        =====API DOCS=====\n        Remove success node.\n\n        :param parent: Primary key of parent node to disassociate success node from.\n        :type parent: int\n        :param child: Primary key of child node to be disassociated.\n        :type child: int\n        :returns: Dictionary of only one key \"changed\", which indicates whether the disassociation succeeded.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        return self._disassoc(\n            self._forward_rel_name('success'), parent, child)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef associate_failure_node(self, parent, child=None, **kwargs):\n        return self._assoc_or_create('failure', parent, child, **kwargs)", "response": "Associate a failure node with a parent node."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nremove a failure node link.", "response": "def disassociate_failure_node(self, parent, child):\n        \"\"\"Remove a failure node link.\n        The resulatant 2 nodes will both become root nodes.\n\n        =====API DOCS=====\n        Remove a failure node link.\n\n        :param parent: Primary key of parent node to disassociate failure node from.\n        :type parent: int\n        :param child: Primary key of child node to be disassociated.\n        :type child: int\n        :returns: Dictionary of only one key \"changed\", which indicates whether the disassociation succeeded.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        return self._disassoc(\n            self._forward_rel_name('failure'), parent, child)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef associate_always_node(self, parent, child=None, **kwargs):\n        return self._assoc_or_create('always', parent, child, **kwargs)", "response": "Associate an always node with a parent node."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndisassociating an always node link from parent and child nodes.", "response": "def disassociate_always_node(self, parent, child):\n        \"\"\"Remove an always node link.\n        The resultant 2 nodes will both become root nodes.\n\n        =====API DOCS=====\n        Remove an always node link.\n\n        :param parent: Primary key of parent node to disassociate always node from.\n        :type parent: int\n        :param child: Primary key of child node to be disassociated.\n        :type child: int\n        :returns: Dictionary of only one key \"changed\", which indicates whether the disassociation succeeded.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        return self._disassoc(\n            self._forward_rel_name('always'), parent, child)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngives some CLI input data return the following and their types.", "response": "def obj_res(data, fail_on=['type', 'obj', 'res']):\n        \"\"\"\n        Given some CLI input data,\n        Returns the following and their types:\n        obj - the role grantee\n        res - the resource that the role applies to\n        \"\"\"\n        errors = []\n        if not data.get('type', None) and 'type' in fail_on:\n            errors += ['You must provide a role type to use this command.']\n\n        # Find the grantee, and remove them from resource_list\n        obj = None\n        obj_type = None\n        for fd in ACTOR_FIELDS:\n            if data.get(fd, False):\n                if not obj:\n                    obj = data[fd]\n                    obj_type = fd\n                else:\n                    errors += ['You can not give a role to a user '\n                               'and team at the same time.']\n                    break\n        if not obj and 'obj' in fail_on:\n            errors += ['You must specify either user or '\n                       'team to use this command.']\n\n        # Out of the resource list, pick out available valid resource field\n        res = None\n        res_type = None\n        for fd in RESOURCE_FIELDS:\n            if data.get(fd, False):\n                if not res:\n                    res = data[fd]\n                    res_type = fd\n                    if res_type == 'target_team':\n                        res_type = 'team'\n                else:\n                    errors += ['You can only give a role to one '\n                               'type of resource at a time.']\n                    break\n        if not res and 'res' in fail_on:\n            errors += ['You must specify a target resource '\n                       'to use this command.']\n\n        if errors:\n            raise exc.UsageError(\"\\n\".join(errors))\n        return obj, obj_type, res, res_type"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef data_endpoint(cls, in_data, ignore=[]):\n        obj, obj_type, res, res_type = cls.obj_res(in_data, fail_on=[])\n        data = {}\n        if 'obj' in ignore:\n            obj = None\n        if 'res' in ignore:\n            res = None\n        # Input fields are not actually present on role model, and all have\n        # to be managed as individual special-cases\n        if obj and obj_type == 'user':\n            data['members__in'] = obj\n        if obj and obj_type == 'team':\n            endpoint = '%s/%s/roles/' % (grammar.pluralize(obj_type), obj)\n            if res is not None:\n                # For teams, this is the best lookup we can do\n                #  without making the additional request for its member_role\n                data['object_id'] = res\n        elif res:\n            endpoint = '%s/%s/object_roles/' % (grammar.pluralize(res_type), res)\n        else:\n            endpoint = '/roles/'\n        if in_data.get('type', False):\n            data['role_field'] = '%s_role' % in_data['type'].lower()\n        # Add back fields unrelated to role lookup, such as all_pages\n        for key, value in in_data.items():\n            if key not in RESOURCE_FIELDS and key not in ['type', 'user', 'team']:\n                data[key] = value\n        return data, endpoint", "response": "Converts a set of CLI input arguments in_data into a dictionary that can be used to look up a specific object or list of roles."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\npopulates the item_dict with the resource_name and resource_type fields.", "response": "def populate_resource_columns(item_dict):\n        \"\"\"Operates on item_dict\n\n        Promotes the resource_name and resource_type fields to the\n        top-level of the serialization so they can be printed as columns.\n        Also makes a copies name field to type, which is a default column.\"\"\"\n        item_dict['type'] = item_dict['name']\n        if len(item_dict['summary_fields']) == 0:\n            # Singleton roles omit these fields\n            item_dict['resource_name'] = None\n            item_dict['resource_type'] = None\n        else:\n            sf = item_dict['summary_fields']\n            # Explination of fallback state:\n            # The situation where resource_name or resource_type is not present\n            # should not be seen for singleton roles, and where it is seen,\n            # there may be a problem with data quality on the server\n            item_dict['resource_name'] = sf.get('resource_name', '[unknown]')\n            item_dict['resource_type'] = sf.get('resource_type', '[unknown]')"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nadding or remove columns from the output.", "response": "def set_display_columns(self, set_true=[], set_false=[]):\n        \"\"\"Add or remove columns from the output.\"\"\"\n        for i in range(len(self.fields)):\n            if self.fields[i].name in set_true:\n                self.fields[i].display = True\n            elif self.fields[i].name in set_false:\n                self.fields[i].display = False"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\npopulating columns and sets display attribute as needed. Operates on data.", "response": "def configure_display(self, data, kwargs=None, write=False):\n        \"\"\"Populates columns and sets display attribute as needed.\n        Operates on data.\"\"\"\n        if settings.format != 'human':\n            return  # This is only used for human format\n        if write:\n            obj, obj_type, res, res_type = self.obj_res(kwargs)\n            data['type'] = kwargs['type']\n            data[obj_type] = obj\n            data[res_type] = res\n            self.set_display_columns(\n                set_false=['team' if obj_type == 'user' else 'user'],\n                set_true=['target_team' if res_type == 'team' else res_type])\n        else:\n            self.set_display_columns(\n                set_false=['user', 'team'],\n                set_true=['resource_name', 'resource_type'])\n            if 'results' in data:\n                for i in range(len(data['results'])):\n                    self.populate_resource_columns(data['results'][i])\n            else:\n                self.populate_resource_columns(data)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef role_write(self, fail_on_found=False, disassociate=False, **kwargs):\n\n        # Get the role, using only the resource data\n        data, self.endpoint = self.data_endpoint(kwargs, ignore=['obj'])\n        debug.log('Checking if role exists.', header='details')\n        response = self.read(pk=None, fail_on_no_results=True,\n                             fail_on_multiple_results=True, **data)\n        role_data = response['results'][0]\n        role_id = role_data['id']\n\n        # Role exists, change display settings to output something\n        self.configure_display(role_data, kwargs, write=True)\n\n        # Check if user/team has this role\n        # Implictly, force_on_exists is false for roles\n        obj, obj_type, res, res_type = self.obj_res(kwargs)\n        debug.log('Checking if %s already has role.' % obj_type,\n                  header='details')\n        data, self.endpoint = self.data_endpoint(kwargs)\n        data['content_type__model'] = res_type.replace('_', '')\n        response = self.read(pk=None, fail_on_no_results=False,\n                             fail_on_multiple_results=False, **data)\n\n        msg = ''\n        if response['count'] > 0 and not disassociate:\n            msg = 'This %s is already a member of the role.' % obj_type\n        elif response['count'] == 0 and disassociate:\n            msg = 'This %s is already a non-member of the role.' % obj_type\n\n        if msg:\n            role_data['changed'] = False\n            if fail_on_found:\n                raise exc.NotFound(msg)\n            else:\n                debug.log(msg, header='DECISION')\n                return role_data\n\n        # Add or remove the user/team to the role\n        debug.log('Attempting to %s the %s in this role.' % (\n            'remove' if disassociate else 'add', obj_type), header='details')\n        post_data = {'id': role_id}\n        if disassociate:\n            post_data['disassociate'] = True\n        client.post('%s/%s/roles/' % (grammar.pluralize(obj_type), obj),\n                    data=post_data)\n        role_data['changed'] = True\n        return role_data", "response": "Re - implementation of the parent write method specific to roles."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a list of objects for the current resource.", "response": "def list(self, **kwargs):\n        \"\"\"Return a list of roles.\n\n        =====API DOCS=====\n        Retrieve a list of objects.\n\n        :param all_pages: Flag that if set, collect all pages of content from the API when returning results.\n        :type all_pages: bool\n        :param page: The page to show. Ignored if all_pages is set.\n        :type page: int\n        :param query: Contains 2-tuples used as query parameters to filter resulting resource objects.\n        :type query: list\n        :param `**kwargs`: Keyword arguments list of available fields used for searching resource objects.\n        :returns: A JSON object containing details of all resource objects returned by Tower backend.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        data, self.endpoint = self.data_endpoint(kwargs)\n        r = super(Resource, self).list(**data)\n\n        # Change display settings and data format for human consumption\n        self.configure_display(r)\n        return r"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef get(self, pk=None, **kwargs):\n        if kwargs.pop('include_debug_header', True):\n            debug.log('Getting the role record.', header='details')\n        data, self.endpoint = self.data_endpoint(kwargs)\n        response = self.read(pk=pk, fail_on_no_results=True,\n                             fail_on_multiple_results=True, **data)\n        item_dict = response['results'][0]\n        self.configure_display(item_dict)\n        return item_dict", "response": "Get information about a resource."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef grant(self, fail_on_found=False, **kwargs):\n        return self.role_write(fail_on_found=fail_on_found, **kwargs)", "response": "Add a user or a team to a role."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nremoves a user or a team from a role.", "response": "def revoke(self, fail_on_found=False, **kwargs):\n        \"\"\"Remove a user or a team from a role. Required information:\n        1) Type of the role\n        2) Resource of the role, inventory, credential, or any other\n        3) A user or a team to add to the role\n\n        =====API DOCS=====\n        Remove a user or a team from a role. Required information:\n        * Type of the role.\n        * Resource of the role, inventory, credential, or any other.\n        * A user or a team to add to the role.\n\n        :param fail_on_found: Flag that if set, the operation fails if a user/team dose not have the role.\n        :type fail_on_found: bool\n        :param `**kwargs`: The user to be disassociated and the role to disassociate.\n        :returns: parsed JSON of role revoke.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        return self.role_write(fail_on_found=fail_on_found,\n                               disassociate=True, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _compare_node_lists(old, new):\n    '''\n    Investigate two lists of workflow TreeNodes and categorize them.\n\n    There will be three types of nodes after categorization:\n        1. Nodes that only exists in the new list. These nodes will later be\n        created recursively.\n        2. Nodes that only exists in the old list. These nodes will later be\n        deleted recursively.\n        3. Node pairs that makes an exact match. These nodes will be further\n        investigated.\n\n    Corresponding nodes of old and new lists will be distinguished by their\n    unified_job_template value. A special case is that both the old and the new\n    lists contain one type of node, say A, and at least one of them contains\n    duplicates. In this case all A nodes in the old list will be categorized as\n    to-be-deleted and all A nodes in the new list will be categorized as\n    to-be-created.\n    '''\n    to_expand = []\n    to_delete = []\n    to_recurse = []\n    old_records = {}\n    new_records = {}\n    for tree_node in old:\n        old_records.setdefault(tree_node.unified_job_template, [])\n        old_records[tree_node.unified_job_template].append(tree_node)\n    for tree_node in new:\n        new_records.setdefault(tree_node.unified_job_template, [])\n        new_records[tree_node.unified_job_template].append(tree_node)\n    for ujt_id in old_records:\n        if ujt_id not in new_records:\n            to_delete.extend(old_records[ujt_id])\n            continue\n        old_list = old_records[ujt_id]\n        new_list = new_records.pop(ujt_id)\n        if len(old_list) == 1 and len(new_list) == 1:\n            to_recurse.append((old_list[0], new_list[0]))\n        else:\n            to_delete.extend(old_list)\n            to_expand.extend(new_list)\n    for nodes in new_records.values():\n        to_expand.extend(nodes)\n    return to_expand, to_delete, to_recurse", "response": "Compares two lists of workflow TreeNodes and returns a list of nodes that are identical."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ntaking the list results from the API in node_results and translates this data into a dictionary organized in heirarchial structure", "response": "def _workflow_node_structure(node_results):\n        '''\n        Takes the list results from the API in `node_results` and\n        translates this data into a dictionary organized in a\n        human-readable heirarchial structure\n        '''\n        # Build list address translation, and create backlink lists\n        node_list_pos = {}\n        for i, node_result in enumerate(node_results):\n            for rel in ['success', 'failure', 'always']:\n                node_result['{0}_backlinks'.format(rel)] = []\n            node_list_pos[node_result['id']] = i\n\n        # Populate backlink lists\n        for node_result in node_results:\n            for rel in ['success', 'failure', 'always']:\n                for sub_node_id in node_result['{0}_nodes'.format(rel)]:\n                    j = node_list_pos[sub_node_id]\n                    node_results[j]['{0}_backlinks'.format(rel)].append(\n                        node_result['id'])\n\n        # Find the root nodes\n        root_nodes = []\n        for node_result in node_results:\n            is_root = True\n            for rel in ['success', 'failure', 'always']:\n                if node_result['{0}_backlinks'.format(rel)] != []:\n                    is_root = False\n                    break\n            if is_root:\n                root_nodes.append(node_result['id'])\n\n        # Create network dictionary recursively from root nodes\n        def branch_schema(node_id):\n            i = node_list_pos[node_id]\n            node_dict = node_results[i]\n            ret_dict = {\"id\": node_id}\n            for fd in NODE_STANDARD_FIELDS:\n                val = node_dict.get(fd, None)\n                if val is not None:\n                    if fd == 'unified_job_template':\n                        job_type = node_dict['summary_fields'][\n                            'unified_job_template']['unified_job_type']\n                        ujt_key = JOB_TYPES[job_type]\n                        ret_dict[ujt_key] = val\n                    else:\n                        ret_dict[fd] = val\n            for rel in ['success', 'failure', 'always']:\n                sub_node_id_list = node_dict['{0}_nodes'.format(rel)]\n                if len(sub_node_id_list) == 0:\n                    continue\n                relationship_name = '{0}_nodes'.format(rel)\n                ret_dict[relationship_name] = []\n                for sub_node_id in sub_node_id_list:\n                    ret_dict[relationship_name].append(\n                        branch_schema(sub_node_id))\n            return ret_dict\n\n        schema_dict = []\n        for root_node_id in root_nodes:\n            schema_dict.append(branch_schema(root_node_id))\n        return schema_dict"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _get_schema(self, wfjt_id):\n        node_res = get_resource('node')\n        node_results = node_res.list(workflow_job_template=wfjt_id,\n                                     all_pages=True)['results']\n        return self._workflow_node_structure(node_results)", "response": "Returns a dictionary that represents the node network of the workflow job template"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns the latest schema of the workflow job template.", "response": "def schema(self, wfjt, node_network=None):\n        \"\"\"\n        Convert YAML/JSON content into workflow node objects if\n        node_network param is given.\n        If not, print a YAML representation of the node network.\n\n        =====API DOCS=====\n        Convert YAML/JSON content into workflow node objects if ``node_network`` param is given. If not,\n        print a YAML representation of the node network.\n\n        :param wfjt: Primary key or name of the workflow job template to run schema against.\n        :type wfjt: str\n        :param node_network: JSON- or YAML-formatted string representing the topology of the workflow job\n                             template be updated to.\n        :type node_network: str\n        :returns: The latest topology (possibly after modification) of the workflow job template.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        existing_network = self._get_schema(wfjt)\n        if not isinstance(existing_network, list):\n            existing_network = []\n        if node_network is None:\n            if settings.format == 'human':\n                settings.format = 'yaml'\n            return existing_network\n\n        if hasattr(node_network, 'read'):\n            node_network = node_network.read()\n        node_network = string_to_dict(\n            node_network, allow_kv=False, require_dict=False)\n        if not isinstance(node_network, list):\n            node_network = []\n\n        _update_workflow([TreeNode(x, wfjt, include_id=True) for x in existing_network],\n                         [TreeNode(x, wfjt) for x in node_network])\n\n        if settings.format == 'human':\n            settings.format = 'yaml'\n        return self._get_schema(wfjt)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef associate_notification_template(self, workflow,\n                                        notification_template, status):\n        \"\"\"Associate a notification template from this workflow.\n\n        =====API DOCS=====\n        Associate a notification template from this workflow job template.\n\n        :param workflow: The workflow job template to associate to.\n        :type workflow: str\n        :param notification_template: The notification template to be associated.\n        :type notification_template: str\n        :param status: type of notification this notification template should be associated to.\n        :type status: str\n        :returns: Dictionary of only one key \"changed\", which indicates whether the association succeeded.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        return self._assoc('notification_templates_%s' % status,\n                           workflow, notification_template)", "response": "Associate a notification template with this workflow job template."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndisassociating a notification template from this workflow job template.", "response": "def disassociate_notification_template(self, workflow,\n                                           notification_template, status):\n        \"\"\"Disassociate a notification template from this workflow.\n\n        =====API DOCS=====\n        Disassociate a notification template from this workflow job template.\n\n        :param job_template: The workflow job template to disassociate from.\n        :type job_template: str\n        :param notification_template: The notification template to be disassociated.\n        :type notification_template: str\n        :param status: type of notification this notification template should be disassociated from.\n        :type status: str\n        :returns: Dictionary of only one key \"changed\", which indicates whether the disassociation succeeded.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        return self._disassoc('notification_templates_%s' % status,\n                              workflow, notification_template)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef create(self, fail_on_found=False, force_on_exists=False, **kwargs):\n        if kwargs.get('parent', None):\n            parent_data = self.set_child_endpoint(parent=kwargs['parent'], inventory=kwargs.get('inventory', None))\n            kwargs['inventory'] = parent_data['inventory']\n        elif 'inventory' not in kwargs:\n            raise exc.UsageError('To create a group, you must provide a parent inventory or parent group.')\n        return super(Resource, self).create(fail_on_found=fail_on_found, force_on_exists=force_on_exists, **kwargs)", "response": "Create a group.\n\n        =====API DOCS=====\n        Create a group.\n\n        :param parent: Primary key or name of the group which will be the parent of created group.\n        :type parent: str\n        :param fail_on_found: Flag that if set, the operation fails if an object matching the unique criteria\n                              already exists.\n        :type fail_on_found: bool\n        :param force_on_exists: Flag that if set, then if a match is found on unique fields, other fields will\n                                be updated to the provided values.; If unset, a match causes the request to be\n                                a no-op.\n        :type force_on_exists: bool\n        :param `**kwargs`: Keyword arguments which, all together, will be used as POST body to create the\n                           resource object.\n        :returns: A dictionary combining the JSON output of the created resource, as well as two extra fields:\n                  \"changed\", a flag indicating if the resource is created successfully; \"id\", an integer which\n                  is the primary key of the created object.\n        :rtype: dict\n        :raises tower_cli.exceptions.UsageError: When inventory is not provided in ``**kwargs`` and ``parent``\n                                                 is not provided.\n\n        =====API DOCS====="}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef list(self, root=False, **kwargs):\n        # Option to list children of a parent group\n        if kwargs.get('parent', None):\n            self.set_child_endpoint(parent=kwargs['parent'], inventory=kwargs.get('inventory', None))\n            kwargs.pop('parent')\n        # Sanity check: If we got `--root` and no inventory, that's an error.\n        if root and not kwargs.get('inventory', None):\n            raise exc.UsageError('The --root option requires specifying an inventory also.')\n        # If we are tasked with getting root groups, do that.\n        if root:\n            inventory_id = kwargs['inventory']\n            r = client.get('/inventories/%d/root_groups/' % inventory_id)\n            return r.json()\n        # Return the superclass implementation.\n        return super(Resource, self).list(**kwargs)", "response": "Return a list of groups."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef associate(self, group, parent, **kwargs):\n        parent_id = self.lookup_with_inventory(parent, kwargs.get('inventory', None))['id']\n        group_id = self.lookup_with_inventory(group, kwargs.get('inventory', None))['id']\n        return self._assoc('children', parent_id, group_id)", "response": "Associate this group with the specified group."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef disassociate(self, group, parent, **kwargs):\n        parent_id = self.lookup_with_inventory(parent, kwargs.get('inventory', None))['id']\n        group_id = self.lookup_with_inventory(group, kwargs.get('inventory', None))['id']\n        return self._disassoc('children', parent_id, group_id)", "response": "Disassociate this group from the specified group."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a dictionary given a string with yaml or json syntax.", "response": "def string_to_dict(var_string, allow_kv=True, require_dict=True):\n    \"\"\"Returns a dictionary given a string with yaml or json syntax.\n    If data is not present in a key: value format, then it return\n    an empty dictionary.\n\n    Attempts processing string by 3 different methods in order:\n        1. as JSON      2. as YAML      3. as custom key=value syntax\n    Throws an error if all of these fail in the standard ways.\"\"\"\n    # try:\n    #     # Accept all valid \"key\":value types of json\n    #     return_dict = json.loads(var_string)\n    #     assert type(return_dict) is dict\n    # except (TypeError, AttributeError, ValueError, AssertionError):\n    try:\n        # Accept all JSON and YAML\n        return_dict = yaml.load(var_string, Loader=yaml.SafeLoader)\n        if require_dict:\n            assert type(return_dict) is dict\n    except (AttributeError, yaml.YAMLError, AssertionError):\n        # if these fail, parse by key=value syntax\n        try:\n            assert allow_kv\n            return_dict = parse_kv(var_string)\n        except Exception:\n            raise exc.TowerCLIError(\n                'failed to parse some of the extra '\n                'variables.\\nvariables: \\n%s' % var_string\n            )\n    return return_dict"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a string that is valid JSON or YAML and contains all the the sgi variables in every extra_vars_opt inside of extra_vars_list.", "response": "def process_extra_vars(extra_vars_list, force_json=True):\n    \"\"\"Returns a string that is valid JSON or YAML and contains all the\n    variables in every extra_vars_opt inside of extra_vars_list.\n\n    Args:\n       parse_kv (bool): whether to allow key=value syntax.\n       force_json (bool): if True, always output json.\n    \"\"\"\n    # Read from all the different sources and put into dictionary\n    extra_vars = {}\n    extra_vars_yaml = \"\"\n    for extra_vars_opt in extra_vars_list:\n        # Load file content if necessary\n        if extra_vars_opt.startswith(\"@\"):\n            with open(extra_vars_opt[1:], 'r') as f:\n                extra_vars_opt = f.read()\n            # Convert text markup to a dictionary conservatively\n            opt_dict = string_to_dict(extra_vars_opt, allow_kv=False)\n        else:\n            # Convert text markup to a dictionary liberally\n            opt_dict = string_to_dict(extra_vars_opt, allow_kv=True)\n        # Rolling YAML-based string combination\n        if any(line.startswith(\"#\") for line in extra_vars_opt.split('\\n')):\n            extra_vars_yaml += extra_vars_opt + \"\\n\"\n        elif extra_vars_opt != \"\":\n            extra_vars_yaml += yaml.dump(\n                opt_dict, default_flow_style=False) + \"\\n\"\n        # Combine dictionary with cumulative dictionary\n        extra_vars.update(opt_dict)\n\n    # Return contents in form of a string\n    if not force_json:\n        try:\n            # Conditions to verify it is safe to return rolling YAML string\n            try_dict = yaml.load(extra_vars_yaml, Loader=yaml.SafeLoader)\n            assert type(try_dict) is dict\n            debug.log('Using unprocessed YAML', header='decision', nl=2)\n            return extra_vars_yaml.rstrip()\n        except Exception:\n            debug.log('Failed YAML parsing, defaulting to JSON',\n                      header='decison', nl=2)\n    if extra_vars == {}:\n        return \"\"\n    return json.dumps(extra_vars, ensure_ascii=False)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndumping a dictionary into a string as parse result of the original data structure.", "response": "def ordered_dump(data, Dumper=yaml.Dumper, **kws):\n    \"\"\"Expand PyYAML's built-in dumper to support parsing OrderedDict. Return\n    a string as parse result of the original data structure, which includes\n    OrderedDict.\n\n    Args:\n        data: the data structure to be dumped(parsed) which is supposed to\n        contain OrderedDict.\n        Dumper: the yaml serializer to be expanded and used.\n        kws: extra key-value arguments to be passed to yaml.dump.\n    \"\"\"\n    class OrderedDumper(Dumper):\n        pass\n\n    def _dict_representer(dumper, data):\n        return dumper.represent_mapping(\n            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,\n            data.items())\n    OrderedDumper.add_representer(OrderedDict,\n                                  _dict_representer)\n    return yaml.dump(data, None, OrderedDumper, **kws)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the appropriate URL prefix to prepend to requests.", "response": "def get_prefix(self, include_version=True):\n        \"\"\"Return the appropriate URL prefix to prepend to requests,\n        based on the host provided in settings.\n        \"\"\"\n        host = settings.host\n        if '://' not in host:\n            host = 'https://%s' % host.strip('/')\n        elif host.startswith('http://') and settings.verify_ssl:\n            raise exc.TowerCLIError(\n                'Can not verify ssl with non-https protocol. Change the '\n                'verify_ssl configuration setting to continue.'\n            )\n        # Validate that we have either an http or https based URL\n        url_pieces = urlparse(host)\n        if url_pieces[0] not in ['http', 'https']:\n            raise exc.ConnectionError('URL must be http(s), {} is not valid'.format(url_pieces[0]))\n\n        prefix = urljoin(host, '/api/')\n        if include_version:\n            # We add the / to the end of {} so that our URL has the ending slash.\n            prefix = urljoin(prefix, \"{}/\".format(CUR_API_VERSION))\n\n        return prefix"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef request(self, method, url, *args, **kwargs):\n\n        # If the URL has the api/vX at the front strip it off\n        # This is common to have if you are extracting a URL from an existing object.\n        # For example, any of the 'related' fields of an object will have this\n        import re\n        url = re.sub(\"^/?api/v[0-9]+/\", \"\", url)\n\n        # Piece together the full URL.\n        use_version = not url.startswith('/o/')\n        url = '%s%s' % (self.get_prefix(use_version), url.lstrip('/'))\n\n        # Ansible Tower expects authenticated requests; add the authentication\n        # from settings if it's provided.\n        kwargs.setdefault(\n            'auth',\n            BasicTowerAuth(\n                settings.username,\n                settings.password,\n                self\n            )\n        )\n\n        # POST and PUT requests will send JSON by default; make this\n        # the content_type by default.  This makes it such that we don't have\n        # to constantly write that in our code, which gets repetitive.\n        headers = kwargs.get('headers', {})\n        if method.upper() in ('PATCH', 'POST', 'PUT'):\n            headers.setdefault('Content-Type', 'application/json')\n            kwargs['headers'] = headers\n\n        # If debugging is on, print the URL and data being sent.\n        debug.log('%s %s' % (method, url), fg='blue', bold=True)\n        if method in ('POST', 'PUT', 'PATCH'):\n            debug.log('Data: %s' % kwargs.get('data', {}),\n                      fg='blue', bold=True)\n        if method == 'GET' or kwargs.get('params', None):\n            debug.log('Params: %s' % kwargs.get('params', {}),\n                      fg='blue', bold=True)\n        debug.log('')\n\n        # If this is a JSON request, encode the data value.\n        if headers.get('Content-Type', '') == 'application/json':\n            kwargs['data'] = json.dumps(kwargs.get('data', {}))\n\n        r = self._make_request(method, url, args, kwargs)\n\n        # Sanity check: Did the server send back some kind of internal error?\n        # If so, bubble this up.\n        if r.status_code >= 500:\n            raise exc.ServerError('The Tower server sent back a server error. '\n                                  'Please try again later.')\n\n        # Sanity check: Did we fail to authenticate properly?\n        # If so, fail out now; this is always a failure.\n        if r.status_code == 401:\n            raise exc.AuthError('Invalid Tower authentication credentials (HTTP 401).')\n\n        # Sanity check: Did we get a forbidden response, which means that\n        # the user isn't allowed to do this? Report that.\n        if r.status_code == 403:\n            raise exc.Forbidden(\"You don't have permission to do that (HTTP 403).\")\n\n        # Sanity check: Did we get a 404 response?\n        # Requests with primary keys will return a 404 if there is no response,\n        # and we want to consistently trap these.\n        if r.status_code == 404:\n            raise exc.NotFound('The requested object could not be found.')\n\n        # Sanity check: Did we get a 405 response?\n        # A 405 means we used a method that isn't allowed. Usually this\n        # is a bad request, but it requires special treatment because the\n        # API sends it as a logic error in a few situations (e.g. trying to\n        # cancel a job that isn't running).\n        if r.status_code == 405:\n            raise exc.MethodNotAllowed(\n                \"The Tower server says you can't make a request with the \"\n                \"%s method to that URL (%s).\" % (method, url),\n            )\n\n        # Sanity check: Did we get some other kind of error?\n        # If so, write an appropriate error message.\n        if r.status_code >= 400:\n            raise exc.BadRequest(\n                'The Tower server claims it was sent a bad request.\\n\\n'\n                '%s %s\\nParams: %s\\nData: %s\\n\\nResponse: %s' %\n                (method, url, kwargs.get('params', None),\n                 kwargs.get('data', None), r.content.decode('utf8'))\n            )\n\n        # Django REST Framework intelligently prints API keys in the\n        # order that they are defined in the models and serializer.\n        #\n        # We want to preserve this behavior when it is possible to do so\n        # with minimal effort, because while the order has no explicit meaning,\n        # we make some effort to order keys in a convenient manner.\n        #\n        # To this end, make this response into an APIResponse subclass\n        # (defined below), which has a `json` method that doesn't lose key\n        # order.\n        r.__class__ = APIResponse\n\n        # Return the response object.\n        return r", "response": "Make a request to the Ansible Tower API and return the response."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _separate(self, kwargs):\n        self._pop_none(kwargs)\n        result = {}\n        for field in Resource.config_fields:\n            if field in kwargs:\n                result[field] = kwargs.pop(field)\n                if field in Resource.json_fields:\n\n                    # If result[field] is not a string we can continue on\n                    if not isinstance(result[field], six.string_types):\n                        continue\n\n                    try:\n                        data = json.loads(result[field])\n                        result[field] = data\n                    except ValueError:\n                        raise exc.TowerCLIError('Provided json file format '\n                                                'invalid. Please recheck.')\n        return result", "response": "Remove None - valued and configuration - related keyworded arguments"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _configuration(self, kwargs, config_item):\n        if 'notification_configuration' not in config_item:\n            if 'notification_type' not in kwargs:\n                return\n            nc = kwargs['notification_configuration'] = {}\n            for field in Resource.configuration[kwargs['notification_type']]:\n                if field not in config_item:\n                    raise exc.TowerCLIError('Required config field %s not'\n                                            ' provided.' % field)\n                else:\n                    nc[field] = config_item[field]\n        else:\n            kwargs['notification_configuration'] = \\\n                    config_item['notification_configuration']", "response": "Combine configuration - related keyworded arguments into a single object."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef create(self, fail_on_found=False, force_on_exists=False, **kwargs):\n        config_item = self._separate(kwargs)\n        jt_id = kwargs.pop('job_template', None)\n        status = kwargs.pop('status', 'any')\n        old_endpoint = self.endpoint\n        if jt_id is not None:\n            jt = get_resource('job_template')\n            jt.get(pk=jt_id)\n            try:\n                nt_id = self.get(**copy.deepcopy(kwargs))['id']\n            except exc.NotFound:\n                pass\n            else:\n                if fail_on_found:\n                    raise exc.TowerCLIError('Notification template already '\n                                            'exists and fail-on-found is '\n                                            'switched on. Please use'\n                                            ' \"associate_notification\" method'\n                                            ' of job_template instead.')\n                else:\n                    debug.log('Notification template already exists, '\n                              'associating with job template.',\n                              header='details')\n                    return jt.associate_notification_template(\n                        jt_id, nt_id, status=status)\n            self.endpoint = '/job_templates/%d/notification_templates_%s/' %\\\n                            (jt_id, status)\n        self._configuration(kwargs, config_item)\n        result = super(Resource, self).create(**kwargs)\n        self.endpoint = old_endpoint\n        return result", "response": "Create a new notification template."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef modify(self, pk=None, create_on_missing=False, **kwargs):\n        # Create the resource if needed.\n        if pk is None and create_on_missing:\n            try:\n                self.get(**copy.deepcopy(kwargs))\n            except exc.NotFound:\n                return self.create(**kwargs)\n\n        # Modify everything except notification type and configuration\n        config_item = self._separate(kwargs)\n        notification_type = kwargs.pop('notification_type', None)\n        debug.log('Modify everything except notification type and'\n                  ' configuration', header='details')\n        part_result = super(Resource, self).\\\n            modify(pk=pk, create_on_missing=create_on_missing, **kwargs)\n\n        # Modify notification type and configuration\n        if notification_type is None or \\\n           notification_type == part_result['notification_type']:\n            for item in part_result['notification_configuration']:\n                if item not in config_item or not config_item[item]:\n                    to_add = part_result['notification_configuration'][item]\n                    if not (to_add == '$encrypted$' and\n                            item in Resource.encrypted_fields):\n                        config_item[item] = to_add\n        if notification_type is None:\n            kwargs['notification_type'] = part_result['notification_type']\n        else:\n            kwargs['notification_type'] = notification_type\n        self._configuration(kwargs, config_item)\n        debug.log('Modify notification type and configuration',\n                  header='details')\n        result = super(Resource, self).\\\n            modify(pk=pk, create_on_missing=create_on_missing, **kwargs)\n\n        # Update 'changed' field to give general changed info\n        if 'changed' in result and 'changed' in part_result:\n            result['changed'] = result['changed'] or part_result['changed']\n        return result", "response": "Modify an existing notification template."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef delete(self, pk=None, fail_on_missing=False, **kwargs):\n        self._separate(kwargs)\n        return super(Resource, self).\\\n            delete(pk=pk, fail_on_missing=fail_on_missing, **kwargs)", "response": "Delete the given object from the resource."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a list of all notification templates.", "response": "def list(self, all_pages=False, **kwargs):\n        \"\"\"Return a list of notification templates.\n\n        Note here configuration-related fields like\n        'notification_configuration' and 'channels' will not be\n        used even provided.\n\n        If one or more filters are provided through keyword arguments,\n        filter the results accordingly.\n\n        If no filters are provided, return all results.\n\n        =====API DOCS=====\n        Retrieve a list of objects.\n\n        :param all_pages: Flag that if set, collect all pages of content from the API when returning results.\n        :type all_pages: bool\n        :param page: The page to show. Ignored if all_pages is set.\n        :type page: int\n        :param query: Contains 2-tuples used as query parameters to filter resulting resource objects.\n        :type query: list\n        :param `**kwargs`: Keyword arguments list of available fields used for searching resource objects.\n        :returns: A JSON object containing details of all resource objects returned by Tower backend.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        self._separate(kwargs)\n        return super(Resource, self).list(all_pages=all_pages, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get(self, pk=None, **kwargs):\n        self._separate(kwargs)\n        return super(Resource, self).get(pk=pk, **kwargs)", "response": "Return one and exactly one notification template."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreads tower - cli config values from the environment if present.", "response": "def config_from_environment():\n    \"\"\"Read tower-cli config values from the environment if present, being\n    careful not to override config values that were explicitly passed in.\n    \"\"\"\n    kwargs = {}\n    for k in CONFIG_OPTIONS:\n        env = 'TOWER_' + k.upper()\n        v = os.getenv(env, None)\n        if v is not None:\n            kwargs[k] = v\n    return kwargs"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\napplies the global options that we desire on every method within the tower - cli command.", "response": "def with_global_options(method):\n    \"\"\"Apply the global options that we desire on every method within\n    tower-cli to the given click command.\n    \"\"\"\n    # Create global options for the Tower host, username, and password.\n    #\n    # These are runtime options that will override the configuration file\n    # settings.\n    method = click.option(\n        '-h', '--tower-host',\n        help='The location of the Ansible Tower host. '\n             'HTTPS is assumed as the protocol unless \"http://\" is explicitly '\n             'provided. This will take precedence over a host provided to '\n             '`tower config`, if any.',\n        required=False, callback=_apply_runtime_setting,\n        is_eager=True,\n        expose_value=False\n    )(method)\n    method = click.option(\n        '-t', '--tower-oauth-token',\n        help='OAuth2 token to use to authenticate to Ansible Tower. '\n             'This will take precedence over a token provided to '\n             '`tower config`, if any.',\n        required=False, callback=_apply_runtime_setting,\n        is_eager=True,\n        expose_value=False\n    )(method)\n    method = click.option(\n        '-u', '--tower-username',\n        help='Username to use to authenticate to Ansible Tower. '\n             'This will take precedence over a username provided to '\n             '`tower config`, if any.',\n        required=False, callback=_apply_runtime_setting,\n        is_eager=True,\n        expose_value=False\n    )(method)\n    method = click.option(\n        '-p', '--tower-password',\n        help='Password to use to authenticate to Ansible Tower. '\n             'This will take precedence over a password provided to '\n             '`tower config`, if any. If value is ASK you will be '\n             'prompted for the password',\n        required=False, callback=_apply_runtime_setting,\n        is_eager=True,\n        expose_value=False\n    )(method)\n\n    # Create a global verbose/debug option.\n    method = click.option(\n        '-f', '--format',\n        help='Output format. The \"human\" format is intended for humans '\n             'reading output on the CLI; the \"json\" and \"yaml\" formats '\n             'provide more data, and \"id\" echos the object id only.',\n        type=click.Choice(['human', 'json', 'yaml', 'id']),\n        required=False, callback=_apply_runtime_setting,\n        is_eager=True,\n        expose_value=False\n    )(method)\n    method = click.option(\n        '-v', '--verbose',\n        default=None,\n        help='Show information about requests being made.',\n        is_flag=True,\n        required=False, callback=_apply_runtime_setting,\n        is_eager=True,\n        expose_value=False\n    )(method)\n    method = click.option(\n        '--description-on',\n        default=None,\n        help='Show description in human-formatted output.',\n        is_flag=True,\n        required=False, callback=_apply_runtime_setting,\n        is_eager=True,\n        expose_value=False\n    )(method)\n\n    # Create a global SSL warning option.\n    method = click.option(\n        '--insecure',\n        default=None,\n        help='Turn off insecure connection warnings. Set config verify_ssl '\n             'to make this permanent.',\n        is_flag=True,\n        required=False, callback=_apply_runtime_setting,\n        is_eager=True,\n        expose_value=False\n    )(method)\n\n    # Create a custom certificate specification option.\n    method = click.option(\n        '--certificate',\n        default=None,\n        help='Path to a custom certificate file that will be used throughout'\n             ' the command. Overwritten by --insecure flag if set.',\n        required=False, callback=_apply_runtime_setting,\n        is_eager=True,\n        expose_value=False\n    )(method)\n\n    method = click.option(\n        '--use-token',\n        default=None,\n        help='Turn on Tower\\'s token-based authentication. No longer supported '\n             'in Tower 3.3 and above.',\n        is_flag=True,\n        required=False, callback=_apply_runtime_setting,\n        is_eager=True,\n        expose_value=False\n    )(method)\n    # Manage the runtime settings context\n    method = runtime_context_manager(method)\n\n    # Okay, we're done adding options; return the method.\n    return method"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef pop_option(function, name):\n    for option in getattr(function, '__click_params__', tuple()):\n        if option.name == name:\n            function.__click_params__.remove(option)", "response": "Removes an option from the list of options applied by the click. option decorator."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreading the configuration from the given file.", "response": "def _read(self, fp, fpname):\n        \"\"\"Read the configuration from the given file.\n\n        If the file lacks any section header, add a [general] section\n        header that encompasses the whole thing.\n        \"\"\"\n        # Attempt to read the file using the superclass implementation.\n        #\n        # Check the permissions of the file we are considering reading\n        # if the file exists and the permissions expose it to reads from\n        # other users, raise a warning\n        if os.path.isfile(fpname):\n            file_permission = os.stat(fpname)\n            if fpname != os.path.join(tower_dir, 'tower_cli.cfg') and (\n                (file_permission.st_mode & stat.S_IRGRP) or\n                (file_permission.st_mode & stat.S_IROTH)\n            ):\n                warnings.warn('File {0} readable by group or others.'\n                              .format(fpname), RuntimeWarning)\n        # If it doesn't work because there's no section header, then\n        # create a section header and call the superclass implementation\n        # again.\n        try:\n            return configparser.ConfigParser._read(self, fp, fpname)\n        except configparser.MissingSectionHeaderError:\n            fp.seek(0)\n            string = '[general]\\n%s' % fp.read()\n            flo = StringIO(string)  # flo == file-like object\n            return configparser.ConfigParser._read(self, flo, fpname)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_or_reset_runtime_param(self, key, value):\n        if self._runtime.has_option('general', key):\n            self._runtime = self._new_parser()\n\n        if value is None:\n            return\n        settings._runtime.set('general', key.replace('tower_', ''),\n                              six.text_type(value))", "response": "Sets the value of the specified key in the current context of the runtime settings."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef launch(self, job_template=None, monitor=False, wait=False,\n               timeout=None, no_input=True, extra_vars=None, **kwargs):\n        \"\"\"Launch a new job based on a job template.\n\n        Creates a new job in Ansible Tower, immediately starts it, and\n        returns back an ID in order for its status to be monitored.\n\n        =====API DOCS=====\n        Launch a new job based on a job template.\n\n        :param job_template: Primary key or name of the job template to launch new job.\n        :type job_template: str\n        :param monitor: Flag that if set, immediately calls ``monitor`` on the newly launched job rather\n                        than exiting with a success.\n        :type monitor: bool\n        :param wait: Flag that if set, monitor the status of the job, but do not print while job is in progress.\n        :type wait: bool\n        :param timeout: If provided with ``monitor`` flag set, this attempt will time out after the given number\n                        of seconds.\n        :type timeout: int\n        :param no_input: Flag that if set, suppress any requests for input.\n        :type no_input: bool\n        :param extra_vars: yaml formatted texts that contains extra variables to pass on.\n        :type extra_vars: array of strings\n        :param diff_mode: Specify diff mode for job template to run.\n        :type diff_mode: bool\n        :param limit: Specify host limit for job template to run.\n        :type limit: str\n        :param tags: Specify tagged actions in the playbook to run.\n        :type tags: str\n        :param skip_tags: Specify tagged actions in the playbook to omit.\n        :type skip_tags: str\n        :param job_type: Specify job type for job template to run.\n        :type job_type: str\n        :param verbosity: Specify verbosity of the playbook run.\n        :type verbosity: int\n        :param inventory: Specify machine credential for job template to run.\n        :type inventory: str\n        :param credential: Specify machine credential for job template to run.\n        :type credential: str\n        :returns: Result of subsequent ``monitor`` call if ``monitor`` flag is on; Result of subsequent\n                  ``wait`` call if ``wait`` flag is on; Result of subsequent ``status`` call if none of\n                  the two flags are on.\n        :rtype: dict\n\n        =====API DOCS=====\n        \"\"\"\n        # Get the job template from Ansible Tower.\n        # This is used as the baseline for starting the job.\n\n        tags = kwargs.get('tags', None)\n        jt_resource = get_resource('job_template')\n        jt = jt_resource.get(job_template)\n\n        # Update the job data by adding an automatically-generated job name,\n        # and removing the ID.\n        data = {}\n        if tags:\n            data['job_tags'] = tags\n\n        # Initialize an extra_vars list that starts with the job template\n        # preferences first, if they exist\n        extra_vars_list = []\n        if 'extra_vars' in data and len(data['extra_vars']) > 0:\n            # But only do this for versions before 2.3\n            debug.log('Getting version of Tower.', header='details')\n            r = client.get('/config/')\n            if LooseVersion(r.json()['version']) < LooseVersion('2.4'):\n                extra_vars_list = [data['extra_vars']]\n\n        # Add the runtime extra_vars to this list\n        if extra_vars:\n            extra_vars_list += list(extra_vars)  # accept tuples\n\n        # If the job template requires prompting for extra variables,\n        # do so (unless --no-input is set).\n        if jt.get('ask_variables_on_launch', False) and not no_input \\\n                and not extra_vars:\n            # If JT extra_vars are JSON, echo them to user as YAML\n            initial = parser.process_extra_vars(\n                [jt['extra_vars']], force_json=False\n            )\n            initial = '\\n'.join((\n                '# Specify extra variables (if any) here as YAML.',\n                '# Lines beginning with \"#\" denote comments.',\n                initial,\n            ))\n            extra_vars = click.edit(initial) or ''\n            if extra_vars != initial:\n                extra_vars_list = [extra_vars]\n\n        # Data is starting out with JT variables, and we only want to\n        # include extra_vars that come from the algorithm here.\n        data.pop('extra_vars', None)\n\n        # Replace/populate data fields if prompted.\n        modified = set()\n        for resource in PROMPT_LIST:\n            if jt.pop('ask_' + resource + '_on_launch', False) and not no_input:\n                resource_object = kwargs.get(resource, None)\n                if type(resource_object) == types.Related:\n                    resource_class = get_resource(resource)\n                    resource_object = resource_class.get(resource).pop('id', None)\n                if resource_object is None:\n                    debug.log('{0} is asked at launch but not provided'.\n                              format(resource), header='warning')\n                elif resource != 'tags':\n                    data[resource] = resource_object\n                    modified.add(resource)\n\n        # Dump extra_vars into JSON string for launching job\n        if len(extra_vars_list) > 0:\n            data['extra_vars'] = parser.process_extra_vars(\n                extra_vars_list, force_json=True\n            )\n\n        # Create the new job in Ansible Tower.\n        start_data = {}\n        endpoint = '/job_templates/%d/launch/' % jt['id']\n        if 'extra_vars' in data and len(data['extra_vars']) > 0:\n            start_data['extra_vars'] = data['extra_vars']\n        if tags:\n            start_data['job_tags'] = data['job_tags']\n        for resource in PROMPT_LIST:\n            if resource in modified:\n                start_data[resource] = data[resource]\n\n        # There's a non-trivial chance that we are going to need some\n        # additional information to start the job; in particular, many jobs\n        # rely on passwords entered at run-time.\n        #\n        # If there are any such passwords on this job, ask for them now.\n        debug.log('Asking for information necessary to start the job.',\n                  header='details')\n        job_start_info = client.get(endpoint).json()\n        for password in job_start_info.get('passwords_needed_to_start', []):\n            start_data[password] = getpass('Password for %s: ' % password)\n\n        # Actually start the job.\n        debug.log('Launching the job.', header='details')\n        self._pop_none(kwargs)\n        kwargs.update(start_data)\n        job_started = client.post(endpoint, data=kwargs)\n\n        # Get the job ID from the result.\n        job_id = job_started.json()['id']\n\n        # If returning json indicates any ignored fields, display it in\n        # verbose mode.\n        if job_started.text == '':\n            ignored_fields = {}\n        else:\n            ignored_fields = job_started.json().get('ignored_fields', {})\n        has_ignored_fields = False\n        for key, value in ignored_fields.items():\n            if value and value != '{}':\n                if not has_ignored_fields:\n                    debug.log('List of ignored fields on the server side:',\n                              header='detail')\n                    has_ignored_fields = True\n                debug.log('{0}: {1}'.format(key, value))\n\n        # Get some information about the running job to print\n        result = self.status(pk=job_id, detail=True)\n        result['changed'] = True\n\n        # If we were told to monitor the job once it started, then call\n        # monitor from here.\n        if monitor:\n            return self.monitor(job_id, timeout=timeout)\n        elif wait:\n            return self.wait(job_id, timeout=timeout)\n\n        return result", "response": "Launch a new job based on a job template."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nlaunching a new ad - hoc command from Ansible Tower.", "response": "def launch(self, monitor=False, wait=False, timeout=None, **kwargs):\n        \"\"\"Launch a new ad-hoc command.\n\n        Runs a user-defined command from Ansible Tower, immediately starts it,\n        and returns back an ID in order for its status to be monitored.\n\n        =====API DOCS=====\n        Launch a new ad-hoc command.\n\n        :param monitor: Flag that if set, immediately calls ``monitor`` on the newly launched command rather\n                        than exiting with a success.\n        :type monitor: bool\n        :param wait: Flag that if set, monitor the status of the job, but do not print while job is in progress.\n        :type wait: bool\n        :param timeout: If provided with ``monitor`` flag set, this attempt will time out after the given number\n                        of seconds.\n        :type timeout: int\n        :param `**kwargs`: Fields needed to create and launch an ad hoc command.\n        :returns: Result of subsequent ``monitor`` call if ``monitor`` flag is on; Result of subsequent ``wait``\n                  call if ``wait`` flag is on; dictionary of \"id\" and \"changed\" if none of the two flags are on.\n        :rtype: dict\n        :raises tower_cli.exceptions.TowerCLIError: When ad hoc commands are not available in Tower backend.\n\n        =====API DOCS=====\n        \"\"\"\n        # This feature only exists for versions 2.2 and up\n        r = client.get('/')\n        if 'ad_hoc_commands' not in r.json():\n            raise exc.TowerCLIError('Your host is running an outdated version'\n                                    'of Ansible Tower that can not run '\n                                    'ad-hoc commands (2.2 or earlier)')\n\n        # Pop the None arguments because we have no .write() method in\n        # inheritance chain for this type of resource. This is needed\n        self._pop_none(kwargs)\n\n        # Actually start the command.\n        debug.log('Launching the ad-hoc command.', header='details')\n        result = client.post(self.endpoint, data=kwargs)\n        command = result.json()\n        command_id = command['id']\n\n        # If we were told to monitor the command once it started, then call\n        # monitor from here.\n        if monitor:\n            return self.monitor(command_id, timeout=timeout)\n        elif wait:\n            return self.wait(command_id, timeout=timeout)\n\n        # Return the command ID and other response data\n        answer = OrderedDict((\n            ('changed', True),\n            ('id', command_id),\n        ))\n        answer.update(result.json())\n        return answer"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nconvert the docstring to more CLI appropriate wording and disambiguate the base class docstrings.", "response": "def _auto_help_text(self, help_text):\n        \"\"\"Given a method with a docstring, convert the docstring\n        to more CLI appropriate wording, and also disambiguate the\n        word \"object\" on the base class docstrings.\n        \"\"\"\n        # Delete API docs if there are any.\n        api_doc_delimiter = '=====API DOCS====='\n        begin_api_doc = help_text.find(api_doc_delimiter)\n        if begin_api_doc >= 0:\n            end_api_doc = help_text.rfind(api_doc_delimiter) + len(api_doc_delimiter)\n            help_text = help_text[:begin_api_doc] + help_text[end_api_doc:]\n        # Convert the word \"object\" to the appropriate type of\n        # object being modified (e.g. user, organization).\n        an_prefix = ('a', 'e', 'i', 'o')\n        if not self.resource_name.lower().startswith(an_prefix):\n            help_text = help_text.replace('an object',\n                                          'a %s' % self.resource_name)\n        if self.resource_name.lower().endswith('y'):\n            help_text = help_text.replace(\n                'objects',\n                '%sies' % self.resource_name[:-1],\n            )\n        help_text = help_text.replace('object', self.resource_name)\n\n        # Convert some common Python terms to their CLI equivalents.\n        help_text = help_text.replace('keyword argument', 'option')\n        help_text = help_text.replace('raise an exception',\n                                      'abort with an error')\n\n        # Convert keyword arguments specified in docstrings enclosed\n        # by backticks to switches.\n        for match in re.findall(r'`([\\w_]+)`', help_text):\n            option = '--%s' % match.replace('_', '-')\n            help_text = help_text.replace('`%s`' % match, option)\n\n        # Done; return the new help text.\n        return help_text"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _echo_method(self, method):\n        @functools.wraps(method)\n        def func(*args, **kwargs):\n            # Echo warning if this method is deprecated.\n            if getattr(method, 'deprecated', False):\n                debug.log('This method is deprecated in Tower 3.0.', header='warning')\n\n            result = method(*args, **kwargs)\n\n            # If this was a request that could result in a modification\n            # of data, print it in Ansible coloring.\n            color_info = {}\n            if isinstance(result, dict) and 'changed' in result:\n                if result['changed']:\n                    color_info['fg'] = 'yellow'\n                else:\n                    color_info['fg'] = 'green'\n\n            # Piece together the result into the proper format.\n            format = getattr(self, '_format_%s' % (getattr(method, 'format_freezer', None) or settings.format))\n            output = format(result)\n\n            # Perform the echo.\n            secho(output, **color_info)\n        return func", "response": "Given a method returns a method that runs the internal\n            method and echos the result."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconvert the payload into a YAML string with proper indentation and return it.", "response": "def _format_yaml(self, payload):\n        \"\"\"Convert the payload into a YAML string with proper\n        indentation and return it.\n        \"\"\"\n        return parser.ordered_dump(payload, Dumper=yaml.SafeDumper,\n                                   default_flow_style=False)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nformats the id of the result.", "response": "def _format_id(self, payload):\n        \"\"\"Echos only the id\"\"\"\n        if 'id' in payload:\n            return str(payload['id'])\n        if 'results' in payload:\n            return ' '.join([six.text_type(item['id']) for item in payload['results']])\n        raise MultipleRelatedError('Could not serialize output with id format.')"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _format_human(self, payload):\n        page = None\n        total_pages = None\n\n        # What are the columns we will show?\n        columns = [field.name for field in self.resource.fields\n                   if field.display or settings.description_on and\n                   field.name == 'description']\n        columns.insert(0, 'id')\n\n        # Save a dictionary-by-name of fields for later use\n        fields_by_name = {}\n        for field in self.resource.fields:\n            fields_by_name[field.name] = field\n\n        # Sanity check: If there is a \"changed\" key in our payload\n        # and little else, we print a short message and not a table.\n        # this specifically applies to deletion\n        if 'changed' in payload and 'id' not in payload:\n            return 'OK. (changed: {0})'.format(\n                six.text_type(payload['changed']).lower(),\n            )\n\n        # Sanity check: If there is no ID and no results, then this\n        # is unusual output; keep our table formatting, but plow\n        # over the columns-as-keys stuff above.\n        # this originally applied to launch/status/update methods\n        # but it may become deprecated\n        if 'id' not in payload and 'results' not in payload:\n            columns = [i for i in payload.keys()]\n\n        # Get our raw rows into a standard format.\n        if 'results' in payload:\n            raw_rows = payload['results']\n            if payload.get('count', 0) > len(payload['results']):\n                prev = payload.get('previous', 0) or 0\n                page = prev + 1\n                count = payload['count']\n                if payload.get('next', None):\n                    total_pages = math.ceil(count / len(raw_rows))\n                else:\n                    total_pages = page\n        else:\n            raw_rows = [payload]\n\n        # If we have no rows to display, return this information\n        # and don't do any further processing.\n        if not raw_rows:\n            return 'No records found.'\n\n        # Determine the width for each column.\n        widths = {}\n        for col in columns:\n            widths[col] = max(\n                len(col),\n                *[len(self.get_print_value(i, col)) for i in raw_rows]\n            )\n            fd = fields_by_name.get(col, None)\n            if fd is not None and fd.col_width is not None:\n                widths[col] = fd.col_width\n\n        # It's possible that the column widths will exceed our terminal\n        # width; if so, reduce column widths accordingly.\n        # TODO: Write this.\n\n        # Put together the divider row.\n        # This is easy and straightforward: it's simply a table divider\n        # using the widths calculated.\n        divider_row = ''\n        for col in columns:\n            divider_row += '=' * widths[col] + ' '\n        divider_row.rstrip()\n\n        # Put together the header row.\n        # This is also easy and straightforward; simply center the\n        # headers (which str.format does for us!).\n        header_row = ''\n        for col in columns:\n            header_row += ('{0:^%d}' % widths[col]).format(col) + ' '\n        header_row.rstrip()\n\n        # Piece together each row of data.\n        data_rows = []\n        for raw_row in raw_rows:\n            data_row = ''\n            for col in columns:\n                template = six.text_type('{0:%d}') % widths[col]\n                value = self.get_print_value(raw_row, col)\n                # Right-align certain native data types\n                if isinstance(raw_row.get(col, 'N/A'), (bool, int)):\n                    template = template.replace('{0:', '{0:>')\n                # Truncate the cell entry if exceeds manually\n                # specified column width limit\n                fd = fields_by_name.get(col, None)\n                if fd is not None and fd.col_width is not None:\n                    str_value = template.format(value or '')\n                    if len(str_value) > fd.col_width:\n                        value = str_value[:fd.col_width]\n                data_row += template.format(value or '') + ' '\n            data_rows.append(data_row.rstrip())\n\n        # Result the resulting table.\n        response = '\\n'.join((\n            divider_row, header_row, divider_row,\n            '\\n'.join(data_rows),\n            divider_row,\n        ))\n        # Don't print page numbers for 1 page results\n        if page and total_pages != 1:\n            response += '(Page %d of %d.)' % (page, total_pages)\n        if payload.get('changed', False):\n            response = 'Resource changed.\\n' + response\n        return response", "response": "Convert the payload into an ASCII table suitable for printing on screen and return it."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget the appropriate method from the Resource and decorate it as a click command and return that method.", "response": "def get_command(self, ctx, name):\n        \"\"\"Retrieve the appropriate method from the Resource,\n        decorate it as a click command, and return that method.\n        \"\"\"\n        # Sanity check: Does a method exist corresponding to this\n        # command? If not, None is returned for click to raise\n        # exception.\n        if not hasattr(self.resource, name):\n            return None\n\n        # Get the method.\n        method = getattr(self.resource, name)\n\n        # Get any attributes that were given at command-declaration\n        # time.\n        attrs = getattr(method, '_cli_command_attrs', {})\n\n        # If the help message comes from the docstring, then\n        # convert it into a message specifically for this resource.\n        help_text = inspect.getdoc(method)\n        attrs['help'] = self._auto_help_text(help_text or '')\n\n        # On some methods, we ignore the defaults, which are intended\n        # for writing and not reading; process this.\n        ignore_defaults = attrs.pop('ignore_defaults', False)\n\n        # Wrap the method, such that it outputs its final return\n        # value rather than returning it.\n        new_method = self._echo_method(method)\n\n        # Soft copy the \"__click_params__\", if any exist.\n        # This is the internal holding method that the click library\n        # uses to store @click.option and @click.argument directives\n        # before the method is converted into a command.\n        #\n        # Because self._echo_method uses @functools.wraps, this is\n        # actually preserved; the purpose of copying it over is\n        # so we can get our resource fields at the top of the help;\n        # the easiest way to do this is to load them in before the\n        # conversion takes place. (This is a happy result of Armin's\n        # work to get around Python's processing decorators\n        # bottom-to-top.)\n        click_params = getattr(method, '__click_params__', [])\n        new_method.__click_params__ = copy(click_params)\n        new_method = with_global_options(new_method)\n\n        # Write options based on the fields available on this resource.\n        fao = attrs.pop('use_fields_as_options', True)\n        if fao:\n            for field in reversed(self.resource.fields):\n                if not field.is_option:\n                    continue\n\n                # If we got an iterable rather than a boolean,\n                # then it is a list of fields to use; check for\n                # presence in that list.\n                if not isinstance(fao, bool) and field.name not in fao:\n                    continue\n\n                # Create the initial arguments based on the\n                # option value. If we have a different key to use\n                # (which is what gets routed to the Tower API),\n                # ensure that is the first argument.\n                args = [field.option]\n                if field.key:\n                    args.insert(0, field.key)\n\n                # short name aliases for common flags\n                short_fields = {\n                    'name': 'n',\n                    'description': 'd',\n                    'inventory': 'i',\n                    'extra_vars': 'e'\n                }\n                if field.name in short_fields:\n                    args.append('-'+short_fields[field.name])\n\n                # Apply the option to the method.\n                option_help = field.help\n                if isinstance(field.type, StructuredInput):\n                    option_help += ' Use @ to get JSON or YAML from a file.'\n                if field.required:\n                    option_help = '[REQUIRED] ' + option_help\n                elif field.read_only:\n                    option_help = '[READ ONLY] ' + option_help\n                option_help = '[FIELD]' + option_help\n                click.option(\n                    *args,\n                    default=field.default if not ignore_defaults else None,\n                    help=option_help,\n                    type=field.type,\n                    show_default=field.show_default,\n                    multiple=field.multiple,\n                    is_eager=False\n                )(new_method)\n\n        # Make a click Command instance using this method\n        # as the callback, and return it.\n        cmd = click.command(name=name, cls=ActionSubcommand, **attrs)(new_method)\n\n        # If this method has a `pk` positional argument,\n        # then add a click argument for it.\n        code = six.get_function_code(method)\n        if 'pk' in code.co_varnames:\n            click.argument('pk', nargs=1, required=False, type=str, metavar='[ID]')(cmd)\n\n        # Done; return the command.\n        return cmd"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef update(self, inventory_source, monitor=False, wait=False,\n               timeout=None, **kwargs):\n        \"\"\"Update the given inventory source.\n\n        =====API DOCS=====\n        Update the given inventory source.\n\n        :param inventory_source: Primary key or name of the inventory source to be updated.\n        :type inventory_source: str\n        :param monitor: Flag that if set, immediately calls ``monitor`` on the newly launched inventory update\n                        rather than exiting with a success.\n        :type monitor: bool\n        :param wait: Flag that if set, monitor the status of the inventory update, but do not print while it is\n                     in progress.\n        :type wait: bool\n        :param timeout: If provided with ``monitor`` flag set, this attempt will time out after the given number\n                        of seconds.\n        :type timeout: int\n        :param `**kwargs`: Fields used to override underlyingl inventory source fields when creating and launching\n                           an inventory update.\n        :returns: Result of subsequent ``monitor`` call if ``monitor`` flag is on; Result of subsequent ``wait``\n                  call if ``wait`` flag is on; dictionary of \"status\" if none of the two flags are on.\n        :rtype: dict\n        :raises tower_cli.exceptions.BadRequest: When the inventory source cannot be updated.\n\n        =====API DOCS=====\n        \"\"\"\n\n        # Establish that we are able to update this inventory source\n        # at all.\n        debug.log('Asking whether the inventory source can be updated.', header='details')\n        r = client.get('%s%d/update/' % (self.endpoint, inventory_source))\n        if not r.json()['can_update']:\n            raise exc.BadRequest('Tower says it cannot run an update against this inventory source.')\n\n        # Run the update.\n        debug.log('Updating the inventory source.', header='details')\n        r = client.post('%s%d/update/' % (self.endpoint, inventory_source), data={})\n        inventory_update_id = r.json()['inventory_update']\n\n        # If we were told to monitor the project update's status, do so.\n        if monitor or wait:\n            if monitor:\n                result = self.monitor(inventory_update_id, parent_pk=inventory_source, timeout=timeout)\n            elif wait:\n                result = self.wait(inventory_update_id, parent_pk=inventory_source, timeout=timeout)\n            inventory = client.get('/inventory_sources/%d/' % result['inventory_source']).json()['inventory']\n            result['inventory'] = int(inventory)\n            return result\n\n        # Done.\n        return {\n            'id': inventory_update_id,\n            'status': 'ok'\n        }", "response": "Update the inventory source."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef list(self, group=None, host_filter=None, **kwargs):\n        if group:\n            kwargs['query'] = kwargs.get('query', ()) + (('groups__in', group),)\n        if host_filter:\n            kwargs['query'] = kwargs.get('query', ()) + (('host_filter', host_filter),)\n        return super(Resource, self).list(**kwargs)", "response": "Return a list of hosts."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef list_facts(self, pk=None, **kwargs):\n        res = self.get(pk=pk, **kwargs)\n        url = self.endpoint + '%d/%s/' % (res['id'], 'ansible_facts')\n        return client.get(url, params={}).json()", "response": "List all available facts of the given host."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nwrite help text for a sub - section of commands.", "response": "def format_command_subsection(self, ctx, formatter, commands, header):\n        \"\"\"Writes help text for a sub-section of commands,\n        specifically to be reused for resource commands\n        and system/configuration commands.\n        \"\"\"\n        rows = []\n        for subcommand in commands:\n            cmd = self.get_command(ctx, subcommand)\n            # What is this, the tool lied about a command.  Ignore it\n            if cmd is None:\n                continue\n\n            help = cmd.short_help or ''\n            rows.append((subcommand, help))\n\n        if rows:\n            with formatter.section(header):\n                formatter.write_dl(rows)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef format_commands(self, ctx, formatter):\n        self.format_command_subsection(\n            ctx, formatter, self.list_misc_commands(), 'Commands'\n        )\n        self.format_command_subsection(\n            ctx, formatter, self.list_resource_commands(), 'Resources'\n        )", "response": "Formats the commands for the current locale."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list of all commands present in the commands and resources folders but not subcommands.", "response": "def list_commands(self, ctx):\n        \"\"\"Return a list of commands present in the commands and resources\n        folders, but not subcommands.\n        \"\"\"\n        commands = set(self.list_resource_commands())\n        commands.union(set(self.list_misc_commands()))\n        return sorted(commands)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a list of multi - commands for each resource type.", "response": "def list_resource_commands(self):\n        \"\"\"Returns a list of multi-commands for each resource type.\n        \"\"\"\n        resource_path = os.path.abspath(os.path.join(\n            os.path.dirname(__file__),\n            os.pardir,\n            'resources'\n        ))\n        answer = set([])\n        for _, name, _ in pkgutil.iter_modules([resource_path]):\n            res = tower_cli.get_resource(name)\n            if not getattr(res, 'internal', False):\n                answer.add(name)\n        return sorted(answer)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef list_misc_commands(self):\n        answer = set([])\n        for cmd_name in misc.__all__:\n            answer.add(cmd_name)\n        return sorted(answer)", "response": "Returns a list of global commands realted to CLI\n        configuration or system management in general."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef get_command(self, ctx, name):\n        # First, attempt to get a basic command from `tower_cli.api.misc`.\n        if name in misc.__all__:\n            return getattr(misc, name)\n\n        # No command was found; try to get a resource.\n        try:\n            resource = tower_cli.get_resource(name)\n            return ResSubcommand(resource)\n        except ImportError:\n            pass\n\n        # Okay, we weren't able to find a command.\n        secho('No such command: %s.' % name, fg='red', bold=True)\n        sys.exit(2)", "response": "Given a command identified by its name import the appropriate\n        module and return the decorated command."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nmarking this method as a CLI command.", "response": "def command(method=None, **kwargs):\n    \"\"\"Mark this method as a CLI command.\n\n    This will only have any meaningful effect in methods that are members of a\n    Resource subclass.\n    \"\"\"\n    # Create the actual decorator to be applied.\n    # This is done in such a way to make `@resources.command`,\n    # `@resources.command()`, and `@resources.command(foo='bar')` all work.\n    def actual_decorator(method):\n        method._cli_command = True\n        method._cli_command_attrs = kwargs\n        return method\n\n    # If we got the method straight-up, apply the decorator and return\n    # the decorated method; otherwise, return the actual decorator for\n    # the Python interpreter to apply.\n    if method and isinstance(method, types.FunctionType):\n        return actual_decorator(method)\n    else:\n        return actual_decorator"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef unified_job_template_options(method):\n    jt_dec = click.option(\n        '--job-template', type=types.Related('job_template'),\n        help='Use this job template as unified_job_template field')\n    prj_dec = click.option(\n        '--project', type=types.Related('project'),\n        help='Use this project as unified_job_template field')\n    inv_src_dec = click.option(\n        '--inventory-source', type=types.Related('inventory_source'),\n        help='Use this inventory source as unified_job_template field')\n\n    def ujt_translation(_method):\n        def _ujt_translation(*args, **kwargs):\n            for fd in ['job_template', 'project', 'inventory_source']:\n                if fd in kwargs and kwargs[fd] is not None:\n                    kwargs['unified_job_template'] = kwargs.pop(fd)\n            return _method(*args, **kwargs)\n        return functools.wraps(_method)(_ujt_translation)\n\n    return ujt_translation(\n        inv_src_dec(\n            prj_dec(\n                jt_dec(\n                    method\n                )\n            )\n        )\n    )", "response": "Adds the decorators for all types of unified job templates and unified job templates in the order they appear."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ngenerate a new state string to be used in authorizations.", "response": "def new_state(self):\n        \"\"\"Generates a state string to be used in authorizations.\"\"\"\n        try:\n            self._state = self.state()\n            log.debug(\"Generated new state %s.\", self._state)\n        except TypeError:\n            self._state = self.state\n            log.debug(\"Re-using previously supplied state %s.\", self._state)\n        return self._state"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nforms an authorization URL.", "response": "def authorization_url(self, url, state=None, **kwargs):\n        \"\"\"Form an authorization URL.\n\n        :param url: Authorization endpoint url, must be HTTPS.\n        :param state: An optional state string for CSRF protection. If not\n                      given it will be generated for you.\n        :param kwargs: Extra parameters to include.\n        :return: authorization_url, state\n        \"\"\"\n        state = state or self.new_state()\n        return (\n            self._client.prepare_request_uri(\n                url,\n                redirect_uri=self.redirect_uri,\n                scope=self.scope,\n                state=state,\n                **kwargs\n            ),\n            state,\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef token_from_fragment(self, authorization_response):\n        self._client.parse_request_uri_response(\n            authorization_response, state=self._state\n        )\n        self.token = self._client.token\n        return self.token", "response": "Parse the token from the URI fragment used by MobileApplicationClients."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nfetches a new access token using a refresh token.", "response": "def refresh_token(\n        self,\n        token_url,\n        refresh_token=None,\n        body=\"\",\n        auth=None,\n        timeout=None,\n        headers=None,\n        verify=True,\n        proxies=None,\n        **kwargs\n    ):\n        \"\"\"Fetch a new access token using a refresh token.\n\n        :param token_url: The token endpoint, must be HTTPS.\n        :param refresh_token: The refresh_token to use.\n        :param body: Optional application/x-www-form-urlencoded body to add the\n                     include in the token request. Prefer kwargs over body.\n        :param auth: An auth tuple or method as accepted by `requests`.\n        :param timeout: Timeout of the request in seconds.\n        :param headers: A dict of headers to be used by `requests`.\n        :param verify: Verify SSL certificate.\n        :param proxies: The `proxies` argument will be passed to `requests`.\n        :param kwargs: Extra parameters to include in the token request.\n        :return: A token dict\n        \"\"\"\n        if not token_url:\n            raise ValueError(\"No token endpoint set for auto_refresh.\")\n\n        if not is_secure_transport(token_url):\n            raise InsecureTransportError()\n\n        refresh_token = refresh_token or self.token.get(\"refresh_token\")\n\n        log.debug(\n            \"Adding auto refresh key word arguments %s.\", self.auto_refresh_kwargs\n        )\n        kwargs.update(self.auto_refresh_kwargs)\n        body = self._client.prepare_refresh_body(\n            body=body, refresh_token=refresh_token, scope=self.scope, **kwargs\n        )\n        log.debug(\"Prepared refresh token request body %s\", body)\n\n        if headers is None:\n            headers = {\n                \"Accept\": \"application/json\",\n                \"Content-Type\": (\"application/x-www-form-urlencoded;charset=UTF-8\"),\n            }\n\n        r = self.post(\n            token_url,\n            data=dict(urldecode(body)),\n            auth=auth,\n            timeout=timeout,\n            headers=headers,\n            verify=verify,\n            withhold_token=True,\n            proxies=proxies,\n        )\n        log.debug(\"Request to refresh token completed with status %s.\", r.status_code)\n        log.debug(\"Response headers were %s and content %s.\", r.headers, r.text)\n        log.debug(\n            \"Invoking %d token response hooks.\",\n            len(self.compliance_hook[\"refresh_token_response\"]),\n        )\n        for hook in self.compliance_hook[\"refresh_token_response\"]:\n            log.debug(\"Invoking hook %s.\", hook)\n            r = hook(r)\n\n        self.token = self._client.parse_request_body_response(r.text, scope=self.scope)\n        if not \"refresh_token\" in self.token:\n            log.debug(\"No new refresh token given. Re-using old.\")\n            self.token[\"refresh_token\"] = refresh_token\n        return self.token"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nintercepts all requests and add the OAuth 2 token if present.", "response": "def request(\n        self,\n        method,\n        url,\n        data=None,\n        headers=None,\n        withhold_token=False,\n        client_id=None,\n        client_secret=None,\n        **kwargs\n    ):\n        \"\"\"Intercept all requests and add the OAuth 2 token if present.\"\"\"\n        if not is_secure_transport(url):\n            raise InsecureTransportError()\n        if self.token and not withhold_token:\n            log.debug(\n                \"Invoking %d protected resource request hooks.\",\n                len(self.compliance_hook[\"protected_request\"]),\n            )\n            for hook in self.compliance_hook[\"protected_request\"]:\n                log.debug(\"Invoking hook %s.\", hook)\n                url, headers, data = hook(url, headers, data)\n\n            log.debug(\"Adding token %s to request.\", self.token)\n            try:\n                url, headers, data = self._client.add_token(\n                    url, http_method=method, body=data, headers=headers\n                )\n            # Attempt to retrieve and save new access token if expired\n            except TokenExpiredError:\n                if self.auto_refresh_url:\n                    log.debug(\n                        \"Auto refresh is set, attempting to refresh at %s.\",\n                        self.auto_refresh_url,\n                    )\n\n                    # We mustn't pass auth twice.\n                    auth = kwargs.pop(\"auth\", None)\n                    if client_id and client_secret and (auth is None):\n                        log.debug(\n                            'Encoding client_id \"%s\" with client_secret as Basic auth credentials.',\n                            client_id,\n                        )\n                        auth = requests.auth.HTTPBasicAuth(client_id, client_secret)\n                    token = self.refresh_token(\n                        self.auto_refresh_url, auth=auth, **kwargs\n                    )\n                    if self.token_updater:\n                        log.debug(\n                            \"Updating token to %s using %s.\", token, self.token_updater\n                        )\n                        self.token_updater(token)\n                        url, headers, data = self._client.add_token(\n                            url, http_method=method, body=data, headers=headers\n                        )\n                    else:\n                        raise TokenUpdated(token)\n                else:\n                    raise\n\n        log.debug(\"Requesting url %s using method %s.\", url, method)\n        log.debug(\"Supplying headers %s and data %s\", headers, data)\n        log.debug(\"Passing through key word arguments %s.\", kwargs)\n        return super(OAuth2Session, self).request(\n            method, url, headers=headers, data=data, **kwargs\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nregisters a new compliance hook for request and response tweaking.", "response": "def register_compliance_hook(self, hook_type, hook):\n        \"\"\"Register a hook for request/response tweaking.\n\n        Available hooks are:\n            access_token_response invoked before token parsing.\n            refresh_token_response invoked before refresh token parsing.\n            protected_request invoked before making a request.\n\n        If you find a new hook is needed please send a GitHub PR request\n        or open an issue.\n        \"\"\"\n        if hook_type not in self.compliance_hook:\n            raise ValueError(\n                \"Hook type %s is not in %s.\", hook_type, self.compliance_hook\n            )\n        self.compliance_hook[hook_type].add(hook)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef authorized(self):\n        if self._client.client.signature_method == SIGNATURE_RSA:\n            # RSA only uses resource_owner_key\n            return bool(self._client.client.resource_owner_key)\n        else:\n            # other methods of authentication use all three pieces\n            return (\n                bool(self._client.client.client_secret)\n                and bool(self._client.client.resource_owner_key)\n                and bool(self._client.client.resource_owner_secret)\n            )", "response": "Boolean that indicates whether this session has an OAuth token\n       ."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef authorization_url(self, url, request_token=None, **kwargs):\n        kwargs[\"oauth_token\"] = request_token or self._client.client.resource_owner_key\n        log.debug(\"Adding parameters %s to url %s\", kwargs, url)\n        return add_params_to_uri(url, kwargs.items())", "response": "Create an authorization URL by appending request_token and optional kwargs to url."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef fetch_request_token(self, url, realm=None, **request_kwargs):\n        self._client.client.realm = \" \".join(realm) if realm else None\n        token = self._fetch_token(url, **request_kwargs)\n        log.debug(\"Resetting callback_uri and realm (not needed in next phase).\")\n        self._client.client.callback_uri = None\n        self._client.client.realm = None\n        return token", "response": "Fetch a request token from the given URL."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef fetch_access_token(self, url, verifier=None, **request_kwargs):\n        if verifier:\n            self._client.client.verifier = verifier\n        if not getattr(self._client.client, \"verifier\", None):\n            raise VerifierMissing(\"No client verifier has been set.\")\n        token = self._fetch_token(url, **request_kwargs)\n        log.debug(\"Resetting verifier attribute, should not be used anymore.\")\n        self._client.client.verifier = None\n        return token", "response": "Fetch an access token from Twitter."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_authorization_response(self, url):\n        log.debug(\"Parsing token from query part of url %s\", url)\n        token = dict(urldecode(urlparse(url).query))\n        log.debug(\"Updating internal client token attribute.\")\n        self._populate_attributes(token)\n        self.token = token\n        return token", "response": "Extract parameters from the post authorization redirect response URL."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef rebuild_auth(self, prepared_request, response):\n        if \"Authorization\" in prepared_request.headers:\n            # If we get redirected to a new host, we should strip out\n            # any authentication headers.\n            prepared_request.headers.pop(\"Authorization\", True)\n            prepared_request.prepare_auth(self.auth)\n        return", "response": "Rebuilds the authentication headers for this request."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ngetting the filepath that the cacher will use.", "response": "def get_fpath(self, cfgstr=None):\n        \"\"\"\n        Reports the filepath that the cacher will use.\n        It will attempt to use '{fname}_{cfgstr}{ext}' unless that is too long.\n        Then cfgstr will be hashed.\n\n        Example:\n            >>> from ubelt.util_cache import Cacher\n            >>> import pytest\n            >>> with pytest.warns(UserWarning):\n            >>>     cacher = Cacher('test_cacher1')\n            >>>     cacher.get_fpath()\n            >>> self = Cacher('test_cacher2', cfgstr='cfg1')\n            >>> self.get_fpath()\n            >>> self = Cacher('test_cacher3', cfgstr='cfg1' * 32)\n            >>> self.get_fpath()\n        \"\"\"\n        condensed = self._condense_cfgstr(cfgstr)\n        fname_cfgstr = '{}_{}{}'.format(self.fname, condensed, self.ext)\n        fpath = join(self.dpath, fname_cfgstr)\n        fpath = normpath(fpath)\n        return fpath"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nyields all existing versions of the type .", "response": "def existing_versions(self):\n        \"\"\"\n        Returns data with different cfgstr values that were previously computed\n        with this cacher.\n\n        Example:\n            >>> from ubelt.util_cache import Cacher\n            >>> # Ensure that some data exists\n            >>> known_fnames = set()\n            >>> cacher = Cacher('versioned_data', cfgstr='1')\n            >>> cacher.ensure(lambda: 'data1')\n            >>> known_fnames.add(cacher.get_fpath())\n            >>> cacher = Cacher('versioned_data', cfgstr='2')\n            >>> cacher.ensure(lambda: 'data2')\n            >>> known_fnames.add(cacher.get_fpath())\n            >>> # List previously computed configs for this type\n            >>> from os.path import basename\n            >>> cacher = Cacher('versioned_data', cfgstr='2')\n            >>> exist_fpaths = set(cacher.existing_versions())\n            >>> exist_fnames = list(map(basename, exist_fpaths))\n            >>> print(exist_fnames)\n            >>> assert exist_fpaths == known_fnames\n\n            ['versioned_data_1.pkl', 'versioned_data_2.pkl']\n        \"\"\"\n        import glob\n        pattern = join(self.dpath, self.fname + '_*' + self.ext)\n        for fname in glob.iglob(pattern):\n            data_fpath = join(self.dpath, fname)\n            yield data_fpath"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef clear(self, cfgstr=None):\n        data_fpath = self.get_fpath(cfgstr)\n        if self.verbose > 0:\n            self.log('[cacher] clear cache')\n        if exists(data_fpath):\n            if self.verbose > 0:\n                self.log('[cacher] removing {}'.format(data_fpath))\n            os.remove(data_fpath)\n\n            # Remove the metadata if it exists\n            meta_fpath = data_fpath + '.meta'\n            if exists(meta_fpath):\n                os.remove(meta_fpath)\n        else:\n            if self.verbose > 0:\n                self.log('[cacher] ... nothing to clear')", "response": "Removes the saved cache and metadata from disk"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ntries to load a cache from a file.", "response": "def tryload(self, cfgstr=None, on_error='raise'):\n        \"\"\"\n        Like load, but returns None if the load fails due to a cache miss.\n\n        Args:\n            on_error (str): How to handle non-io errors errors. Either raise,\n                which re-raises the exception, or clear which deletes the cache\n                and returns None.\n        \"\"\"\n        cfgstr = self._rectify_cfgstr(cfgstr)\n        if self.enabled:\n            try:\n                if self.verbose > 1:\n                    self.log('[cacher] tryload fname={}'.format(self.fname))\n                return self.load(cfgstr)\n            except IOError:\n                if self.verbose > 0:\n                    self.log('[cacher] ... {} cache miss'.format(self.fname))\n            except Exception:\n                if self.verbose > 0:\n                    self.log('[cacher] ... failed to load')\n                if on_error == 'raise':\n                    raise\n                elif on_error == 'clear':\n                    self.clear(cfgstr)\n                    return None\n                else:\n                    raise KeyError('Unknown method on_error={}'.format(on_error))\n        else:\n            if self.verbose > 1:\n                self.log('[cacher] ... cache disabled: fname={}'.format(self.fname))\n        return None"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef load(self, cfgstr=None):\n        from six.moves import cPickle as pickle\n        cfgstr = self._rectify_cfgstr(cfgstr)\n\n        dpath = self.dpath\n        fname = self.fname\n        verbose = self.verbose\n\n        if not self.enabled:\n            if verbose > 1:\n                self.log('[cacher] ... cache disabled: fname={}'.format(self.fname))\n            raise IOError(3, 'Cache Loading Is Disabled')\n\n        fpath = self.get_fpath(cfgstr=cfgstr)\n\n        if not exists(fpath):\n            if verbose > 2:\n                self.log('[cacher] ... cache does not exist: '\n                         'dpath={} fname={} cfgstr={}'.format(\n                             basename(dpath), fname, cfgstr))\n            raise IOError(2, 'No such file or directory: %r' % (fpath,))\n        else:\n            if verbose > 3:\n                self.log('[cacher] ... cache exists: '\n                         'dpath={} fname={} cfgstr={}'.format(\n                             basename(dpath), fname, cfgstr))\n        try:\n            with open(fpath, 'rb') as file_:\n                data = pickle.load(file_)\n        except Exception as ex:\n            if verbose > 0:\n                self.log('CORRUPTED? fpath = %s' % (fpath,))\n            if verbose > 1:\n                self.log('[cacher] ... CORRUPTED? dpath={} cfgstr={}'.format(\n                    basename(dpath), cfgstr))\n            if isinstance(ex, (EOFError, IOError, ImportError)):\n                raise IOError(str(ex))\n            else:\n                if verbose > 1:\n                    self.log('[cacher] ... unknown reason for exception')\n                raise\n        else:\n            if self.verbose > 2:\n                self.log('[cacher] ... {} cache hit'.format(self.fname))\n            elif verbose > 1:\n                self.log('[cacher] ... cache hit')\n        return data", "response": "Loads the data from the cache."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nwrite data to the cache file specified by self. fpath.", "response": "def save(self, data, cfgstr=None):\n        \"\"\"\n        Writes data to path specified by `self.fpath(cfgstr)`.\n\n        Metadata containing information about the cache will also be appended\n        to an adjacent file with the `.meta` suffix.\n\n        Example:\n            >>> from ubelt.util_cache import *  # NOQA\n            >>> # Normal functioning\n            >>> cfgstr = 'long-cfg' * 32\n            >>> cacher = Cacher('test_enabled_save', cfgstr)\n            >>> cacher.save('data')\n            >>> assert exists(cacher.get_fpath()), 'should be enabeled'\n            >>> assert exists(cacher.get_fpath() + '.meta'), 'missing metadata'\n            >>> # Setting the cacher as enabled=False turns it off\n            >>> cacher2 = Cacher('test_disabled_save', 'params', enabled=False)\n            >>> cacher2.save('data')\n            >>> assert not exists(cacher2.get_fpath()), 'should be disabled'\n        \"\"\"\n        from six.moves import cPickle as pickle\n        if not self.enabled:\n            return\n        if self.verbose > 0:\n            self.log('[cacher] ... {} cache save'.format(self.fname))\n\n        cfgstr = self._rectify_cfgstr(cfgstr)\n        condensed = self._condense_cfgstr(cfgstr)\n\n        # Make sure the cache directory exists\n        util_path.ensuredir(self.dpath)\n\n        data_fpath = self.get_fpath(cfgstr=cfgstr)\n        meta_fpath = data_fpath + '.meta'\n\n        # Also save metadata file to reconstruct hashing\n        with open(meta_fpath, 'a') as file_:\n            # TODO: maybe append this in json or YML format?\n            file_.write('\\n\\nsaving {}\\n'.format(util_time.timestamp()))\n            file_.write(self.fname + '\\n')\n            file_.write(condensed + '\\n')\n            file_.write(cfgstr + '\\n')\n            file_.write(str(self.meta) + '\\n')\n\n        with open(data_fpath, 'wb') as file_:\n            # Use protocol 2 to support python2 and 3\n            pickle.dump(data, file_, protocol=self.protocol)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef ensure(self, func, *args, **kwargs):\n        data = self.tryload()\n        if data is None:\n            data = func(*args, **kwargs)\n            self.save(data)\n        return data", "response": "r Wraps around a function that will compute data on cache miss\n           "}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the stamp certificate if it exists otherwise returns None", "response": "def _get_certificate(self, cfgstr=None):\n        \"\"\"\n        Returns the stamp certificate if it exists\n        \"\"\"\n        certificate = self.cacher.tryload(cfgstr=cfgstr)\n        return certificate"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns a list of products", "response": "def _rectify_products(self, product=None):\n        \"\"\" puts products in a normalized format \"\"\"\n        products = self.product if product is None else product\n        if products is None:\n            return None\n        if not isinstance(products, (list, tuple)):\n            products = [products]\n        return products"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nget the hash of the product file", "response": "def _product_file_hash(self, product=None):\n        \"\"\"\n        Get the hash of the each product file\n        \"\"\"\n        if self.hasher is None:\n            return None\n        else:\n            products = self._rectify_products(product)\n            product_file_hash = [\n                util_hash.hash_file(p, hasher=self.hasher, base='hex')\n                for p in products\n            ]\n            return product_file_hash"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncheck to see if a previously existing stamp is still valid and if the expected result of that computation still exists.", "response": "def expired(self, cfgstr=None, product=None):\n        \"\"\"\n        Check to see if a previously existing stamp is still valid and if the\n        expected result of that computation still exists.\n\n        Args:\n            cfgstr (str, optional): override the default cfgstr if specified\n            product (PathLike or Sequence[PathLike], optional): override the\n                default product if specified\n        \"\"\"\n        products = self._rectify_products(product)\n        certificate = self._get_certificate(cfgstr=cfgstr)\n        if certificate is None:\n            # We dont have a certificate, so we are expired\n            is_expired = True\n        elif products is None:\n            # We dont have a product to check, so assume not expired\n            is_expired = False\n        elif not all(map(os.path.exists, products)):\n            # We are expired if the expected product does not exist\n            is_expired = True\n        else:\n            # We are expired if the hash of the existing product data\n            # does not match the expected hash in the certificate\n            product_file_hash = self._product_file_hash(products)\n            certificate_hash = certificate.get('product_file_hash', None)\n            is_expired = product_file_hash != certificate_hash\n        return is_expired"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef renew(self, cfgstr=None, product=None):\n        products = self._rectify_products(product)\n        certificate = {\n            'timestamp': util_time.timestamp(),\n            'product': products,\n        }\n        if products is not None:\n            if not all(map(os.path.exists, products)):\n                raise IOError(\n                    'The stamped product must exist: {}'.format(products))\n            certificate['product_file_hash'] = self._product_file_hash(products)\n        self.cacher.save(certificate, cfgstr=cfgstr)\n        return certificate", "response": "Renews the current entry in the cache."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef isatty(self):  # nocover\n        return (self.redirect is not None and\n                hasattr(self.redirect, 'isatty') and self.redirect.isatty())", "response": "Returns true if the redirect is a terminal."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef encoding(self):\n        if self.redirect is not None:\n            return self.redirect.encoding\n        else:\n            return super(TeeStringIO, self).encoding", "response": "Gets the encoding of the redirect IO object."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nwrite to this and the redirected stream", "response": "def write(self, msg):\n        \"\"\"\n        Write to this and the redirected stream\n        \"\"\"\n        if self.redirect is not None:\n            self.redirect.write(msg)\n        if six.PY2:\n            from xdoctest.utils.util_str import ensure_unicode\n            msg = ensure_unicode(msg)\n        super(TeeStringIO, self).write(msg)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nflush to this and the redirected stream", "response": "def flush(self):  # nocover\n        \"\"\"\n        Flush to this and the redirected stream\n        \"\"\"\n        if self.redirect is not None:\n            self.redirect.flush()\n        super(TeeStringIO, self).flush()"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef log_part(self):\n        self.cap_stdout.seek(self._pos)\n        text = self.cap_stdout.read()\n        self._pos = self.cap_stdout.tell()\n        self.parts.append(text)\n        self.text = text", "response": "Log what has been captured so far"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the path to the data dir used by the current operating system", "response": "def platform_data_dir():\n    \"\"\"\n    Returns path for user-specific data files\n\n    Returns:\n        PathLike : path to the data dir used by the current operating system\n    \"\"\"\n    if LINUX:  # nocover\n        dpath_ = os.environ.get('XDG_DATA_HOME', '~/.local/share')\n    elif DARWIN:  # nocover\n        dpath_  = '~/Library/Application Support'\n    elif WIN32:  # nocover\n        dpath_ = os.environ.get('APPDATA', '~/AppData/Roaming')\n    else:  # nocover\n        raise '~/AppData/Local'\n    dpath = normpath(expanduser(dpath_))\n    return dpath"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef platform_config_dir():\n    if LINUX:  # nocover\n        dpath_ = os.environ.get('XDG_CONFIG_HOME', '~/.config')\n    elif DARWIN:  # nocover\n        dpath_  = '~/Library/Application Support'\n    elif WIN32:  # nocover\n        dpath_ = os.environ.get('APPDATA', '~/AppData/Roaming')\n    else:  # nocover\n        raise NotImplementedError('Unknown Platform  %r' % (sys.platform,))\n    dpath = normpath(expanduser(dpath_))\n    return dpath", "response": "Returns a directory which should be writable for any application\n   "}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nensures that the data directory exists.", "response": "def ensure_app_data_dir(appname, *args):\n    \"\"\"\n    Calls `get_app_data_dir` but ensures the directory exists.\n\n    Args:\n        appname (str): the name of the application\n        *args: any other subdirectories may be specified\n\n    SeeAlso:\n        get_app_data_dir\n\n    Example:\n        >>> import ubelt as ub\n        >>> dpath = ub.ensure_app_data_dir('ubelt')\n        >>> assert exists(dpath)\n    \"\"\"\n    from ubelt import util_path\n    dpath = get_app_data_dir(appname, *args)\n    util_path.ensuredir(dpath)\n    return dpath"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nensure that the application config directory exists.", "response": "def ensure_app_config_dir(appname, *args):\n    \"\"\"\n    Calls `get_app_config_dir` but ensures the directory exists.\n\n    Args:\n        appname (str): the name of the application\n        *args: any other subdirectories may be specified\n\n    SeeAlso:\n        get_app_config_dir\n\n    Example:\n        >>> import ubelt as ub\n        >>> dpath = ub.ensure_app_config_dir('ubelt')\n        >>> assert exists(dpath)\n    \"\"\"\n    from ubelt import util_path\n    dpath = get_app_config_dir(appname, *args)\n    util_path.ensuredir(dpath)\n    return dpath"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nensures that the application cache directory exists.", "response": "def ensure_app_cache_dir(appname, *args):\n    \"\"\"\n    Calls `get_app_cache_dir` but ensures the directory exists.\n\n    Args:\n        appname (str): the name of the application\n        *args: any other subdirectories may be specified\n\n    SeeAlso:\n        get_app_cache_dir\n\n    Example:\n        >>> import ubelt as ub\n        >>> dpath = ub.ensure_app_cache_dir('ubelt')\n        >>> assert exists(dpath)\n    \"\"\"\n    from ubelt import util_path\n    dpath = get_app_cache_dir(appname, *args)\n    util_path.ensuredir(dpath)\n    return dpath"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef startfile(fpath, verbose=True):  # nocover\n    from ubelt import util_cmd\n    if verbose:\n        print('[ubelt] startfile(\"{}\")'.format(fpath))\n    fpath = normpath(fpath)\n    if not exists(fpath):\n        raise Exception('Cannot start nonexistant file: %r' % fpath)\n    if not WIN32:\n        import pipes\n        fpath = pipes.quote(fpath)\n    if LINUX:\n        info = util_cmd.cmd(('xdg-open', fpath), detach=True, verbose=verbose)\n    elif DARWIN:\n        info = util_cmd.cmd(('open', fpath), detach=True, verbose=verbose)\n    elif WIN32:\n        os.startfile(fpath)\n        info = None\n    else:\n        raise RuntimeError('Unknown Platform')\n    if info is not None:\n        if not info['proc']:\n            raise Exception('startfile failed')", "response": "Start a file in the cache."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfinding a command in the local filesystem and return the first matching file with executable permission.", "response": "def find_exe(name, multi=False, path=None):\n    \"\"\"\n    Locate a command.\n\n    Search your local filesystem for an executable and return the first\n    matching file with executable permission.\n\n    Args:\n        name (str): globstr of matching filename\n\n        multi (bool): if True return all matches instead of just the first.\n            Defaults to False.\n\n        path (str or Iterable[PathLike]): overrides the system PATH variable.\n\n    Returns:\n        PathLike or List[PathLike] or None: returns matching executable(s).\n\n    SeeAlso:\n        shutil.which - which is available in Python 3.3+.\n\n    Notes:\n        This is essentially the `which` UNIX command\n\n    References:\n        https://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028\n        https://docs.python.org/dev/library/shutil.html#shutil.which\n\n    Example:\n        >>> find_exe('ls')\n        >>> find_exe('ping')\n        >>> assert find_exe('which') == find_exe(find_exe('which'))\n        >>> find_exe('which', multi=True)\n        >>> find_exe('ping', multi=True)\n        >>> find_exe('cmake', multi=True)\n        >>> find_exe('nvcc', multi=True)\n        >>> find_exe('noexist', multi=True)\n\n    Example:\n        >>> assert not find_exe('noexist', multi=False)\n        >>> assert find_exe('ping', multi=False)\n        >>> assert not find_exe('noexist', multi=True)\n        >>> assert find_exe('ping', multi=True)\n\n    Benchmark:\n        >>> # xdoctest: +IGNORE_WANT\n        >>> import ubelt as ub\n        >>> import shutil\n        >>> for timer in ub.Timerit(100, bestof=10, label='ub.find_exe'):\n        >>>     ub.find_exe('which')\n        >>> for timer in ub.Timerit(100, bestof=10, label='shutil.which'):\n        >>>     shutil.which('which')\n        Timed best=58.71 \u00b5s, mean=59.64 \u00b1 0.96 \u00b5s for ub.find_exe\n        Timed best=72.75 \u00b5s, mean=73.07 \u00b1 0.22 \u00b5s for shutil.which\n    \"\"\"\n    candidates = find_path(name, path=path, exact=True)\n    mode = os.X_OK | os.F_OK\n    results = (fpath for fpath in candidates\n               if os.access(fpath, mode) and not isdir(fpath))\n    if not multi:\n        for fpath in results:\n            return fpath\n    else:\n        return list(results)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef find_path(name, path=None, exact=False):\n    path = os.environ.get('PATH', os.defpath) if path is None else path\n    dpaths = path.split(os.pathsep) if isinstance(path, six.string_types) else path\n    candidates = (join(dpath, name) for dpath in dpaths)\n    if exact:\n        if WIN32:  # nocover\n            pathext = [''] + os.environ.get('PATHEXT', '').split(os.pathsep)\n            candidates = (p + ext for p in candidates for ext in pathext)\n        candidates = filter(exists, candidates)\n    else:\n        import glob\n        candidates = it.chain.from_iterable(\n            glob.glob(pattern) for pattern in candidates)\n    return candidates", "response": "Search for a file or directory on your local filesystem by name."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef editfile(fpath, verbose=True):  # nocover\n    from six import types\n    from ubelt import util_cmd\n    import warnings\n    warnings.warn('Please use xdev.editfile instead', DeprecationWarning)\n    if not isinstance(fpath, six.string_types):\n        if isinstance(fpath, types.ModuleType):\n            fpath = fpath.__file__\n        else:\n            fpath =  sys.modules[fpath.__module__].__file__\n        fpath_py = fpath.replace('.pyc', '.py')\n        if exists(fpath_py):\n            fpath = fpath_py\n\n    if verbose:\n        print('[ubelt] editfile(\"{}\")'.format(fpath))\n\n    editor = os.environ.get('VISUAL', 'gvim')\n\n    if not exists(fpath):\n        raise IOError('Cannot start nonexistant file: %r' % fpath)\n    util_cmd.cmd([editor, fpath], fpath, detach=True)", "response": "Edit a file or code corresponding to a live python object in a new language."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef augpath(path, suffix='', prefix='', ext=None, base=None, multidot=False):\n    # Breakup path\n    dpath, fname = split(path)\n    if multidot:\n        parts = fname.split('.')\n        fname_noext = '.'.join(parts[:1])\n        orig_ext = '.'.join(parts[1:])\n    else:\n        fname_noext, orig_ext = splitext(fname)\n    ext = orig_ext if ext is None else ext\n    fname_noext = fname_noext if base is None else base\n    # Augment and recombine into new path\n    new_fname = ''.join((prefix, fname_noext, suffix, ext))\n    newpath = join(dpath, new_fname)\n    return newpath", "response": "Augments a path with a new basename extension prefix and or suffix."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef userhome(username=None):\n    if username is None:\n        # get home directory for the current user\n        if 'HOME' in os.environ:\n            userhome_dpath = os.environ['HOME']\n        else:  # nocover\n            if sys.platform.startswith('win32'):\n                # win32 fallback when HOME is not defined\n                if 'USERPROFILE' in os.environ:\n                    userhome_dpath = os.environ['USERPROFILE']\n                elif 'HOMEPATH' in os.environ:\n                    drive = os.environ.get('HOMEDRIVE', '')\n                    userhome_dpath = join(drive, os.environ['HOMEPATH'])\n                else:\n                    raise OSError(\"Cannot determine the user's home directory\")\n            else:\n                # posix fallback when HOME is not defined\n                import pwd\n                userhome_dpath = pwd.getpwuid(os.getuid()).pw_dir\n    else:\n        # A specific user directory was requested\n        if sys.platform.startswith('win32'):  # nocover\n            # get the directory name for the current user\n            c_users = dirname(userhome())\n            userhome_dpath = join(c_users, username)\n            if not exists(userhome_dpath):\n                raise KeyError('Unknown user: {}'.format(username))\n        else:\n            import pwd\n            try:\n                pwent = pwd.getpwnam(username)\n            except KeyError:  # nocover\n                raise KeyError('Unknown user: {}'.format(username))\n            userhome_dpath = pwent.pw_dir\n    return userhome_dpath", "response": "Returns the user s home directory."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning the path in system file structure with the given home directory.", "response": "def compressuser(path, home='~'):\n    \"\"\"\n    Inverse of `os.path.expanduser`\n\n    Args:\n        path (PathLike): path in system file structure\n        home (str): symbol used to replace the home path. Defaults to '~', but\n            you might want to use '$HOME' or '%USERPROFILE%' instead.\n\n    Returns:\n        PathLike: path: shortened path replacing the home directory with a tilde\n\n    CommandLine:\n        xdoctest -m ubelt.util_path compressuser\n\n    Example:\n        >>> path = expanduser('~')\n        >>> assert path != '~'\n        >>> assert compressuser(path) == '~'\n        >>> assert compressuser(path + '1') == path + '1'\n        >>> assert compressuser(path + '/1') == join('~', '1')\n        >>> assert compressuser(path + '/1', '$HOME') == join('$HOME', '1')\n    \"\"\"\n    path = normpath(path)\n    userhome_dpath = userhome()\n    if path.startswith(userhome_dpath):\n        if len(path) == len(userhome_dpath):\n            path = home\n        elif path[len(userhome_dpath)] == os.path.sep:\n            path = home + path[len(userhome_dpath):]\n    return path"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef truepath(path, real=False):\n    path = expanduser(path)\n    path = expandvars(path)\n    if real:\n        path = realpath(path)\n    else:\n        path = abspath(path)\n    path = normpath(path)\n    return path", "response": "Returns a string representation of a path that is a true path."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef ensuredir(dpath, mode=0o1777, verbose=None):\n    if verbose is None:  # nocover\n        verbose = 0\n    if isinstance(dpath, (list, tuple)):  # nocover\n        dpath = join(*dpath)\n    if not exists(dpath):\n        if verbose:  # nocover\n            print('Ensuring new directory (%r)' % dpath)\n        if sys.version_info.major == 2:  # nocover\n            os.makedirs(normpath(dpath), mode=mode)\n        else:\n            os.makedirs(normpath(dpath), mode=mode, exist_ok=True)\n    else:\n        if verbose:  # nocover\n            print('Ensuring existing directory (%r)' % dpath)\n    return dpath", "response": "r Ensures that the directory dpath exists. Creates new dir with sticky bits by\n            default."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nparses the description in the README file and return the text.", "response": "def parse_description():\n    \"\"\"\n    Parse the description in the README file\n\n    CommandLine:\n        pandoc --from=markdown --to=rst --output=README.rst README.md\n        python -c \"import setup; print(setup.parse_description())\"\n    \"\"\"\n    from os.path import dirname, join, exists\n    readme_fpath = join(dirname(__file__), 'README.rst')\n    # This breaks on pip install, so check that it exists.\n    if exists(readme_fpath):\n        with open(readme_fpath, 'r') as f:\n            text = f.read()\n        return text\n    return ''"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing a requirements. txt file and return a list of packages that are available on the system.", "response": "def parse_requirements_alt(fname='requirements.txt'):\n    \"\"\"\n    pip install requirements-parser\n    fname='requirements.txt'\n    \"\"\"\n    import requirements\n    from os.path import dirname, join, exists\n    require_fpath = join(dirname(__file__), fname)\n    if exists(require_fpath):\n        # Dont use until this handles platform specific dependencies\n        with open(require_fpath, 'r') as file:\n            requires = list(requirements.parse(file))\n        packages = [r.name for r in requires]\n        return packages\n    return []"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nparsing the requirements file and return a list of dictionaries.", "response": "def parse_requirements(fname='requirements.txt'):\n    \"\"\"\n    Parse the package dependencies listed in a requirements file but strips\n    specific versioning information.\n\n    TODO:\n        perhaps use https://github.com/davidfischer/requirements-parser instead\n\n    CommandLine:\n        python -c \"import setup; print(setup.parse_requirements())\"\n    \"\"\"\n    from os.path import dirname, join, exists\n    import re\n    require_fpath = join(dirname(__file__), fname)\n\n    def parse_line(line):\n        \"\"\"\n        Parse information from a line in a requirements text file\n        \"\"\"\n        info = {}\n        if line.startswith('-e '):\n            info['package'] = line.split('#egg=')[1]\n        else:\n            # Remove versioning from the package\n            pat = '(' + '|'.join(['>=', '==', '>']) + ')'\n            parts = re.split(pat, line, maxsplit=1)\n            parts = [p.strip() for p in parts]\n\n            info['package'] = parts[0]\n            if len(parts) > 1:\n                op, rest = parts[1:]\n                if ';' in rest:\n                    # Handle platform specific dependencies\n                    # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies\n                    version, platform_deps = map(str.strip, rest.split(';'))\n                    info['platform_deps'] = platform_deps\n                else:\n                    version = rest  # NOQA\n                info['version'] = (op, version)\n        return info\n\n    # This breaks on pip install, so check that it exists.\n    if exists(require_fpath):\n        with open(require_fpath, 'r') as f:\n            packages = []\n            for line in f.readlines():\n                line = line.strip()\n                if line and not line.startswith('#'):\n                    info = parse_line(line)\n                    package = info['package']\n                    if not sys.version.startswith('3.4'):\n                        # apparently package_deps are broken in 3.4\n                        platform_deps = info.get('platform_deps')\n                        if platform_deps is not None:\n                            package += ';' + platform_deps\n                    packages.append(package)\n            return packages\n    return []"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef inject_method(self, func, name=None):\n    # TODO: if func is a bound method we should probably unbind it\n    new_method = func.__get__(self, self.__class__)\n    if name is None:\n        name = func.__name__\n    setattr(self, name, new_method)", "response": "Injects a function into an object instance as a bound method."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef writeto(fpath, to_write, aslines=False, verbose=None):\n    if verbose:\n        print('Writing to text file: %r ' % (fpath,))\n\n    with open(fpath, 'wb') as file:\n        if aslines:\n            to_write = map(_ensure_bytes , to_write)\n            file.writelines(to_write)\n        else:\n            # convert to bytes for writing\n            bytes = _ensure_bytes(to_write)\n            file.write(bytes)", "response": "r Writes text to a file."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef readfrom(fpath, aslines=False, errors='replace', verbose=None):\n    if verbose:\n        print('Reading text file: %r ' % (fpath,))\n    if not exists(fpath):\n        raise IOError('File %r does not exist' % (fpath,))\n    with open(fpath, 'rb') as file:\n        if aslines:\n            text = [line.decode('utf8', errors=errors)\n                    for line in file.readlines()]\n            if sys.platform.startswith('win32'):  # nocover\n                # fix line endings on windows\n                text = [\n                    line[:-2] + '\\n' if line.endswith('\\r\\n') else line\n                    for line in text\n                ]\n        else:\n            text = file.read().decode('utf8', errors=errors)\n    return text", "response": "Reads text from a file."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef touch(fpath, mode=0o666, dir_fd=None, verbose=0, **kwargs):\n    if verbose:\n        print('Touching file {}'.format(fpath))\n    if six.PY2:  # nocover\n        with open(fpath, 'a'):\n            os.utime(fpath, None)\n    else:\n        flags = os.O_CREAT | os.O_APPEND\n        with os.fdopen(os.open(fpath, flags=flags, mode=mode, dir_fd=dir_fd)) as f:\n            os.utime(f.fileno() if os.utime in os.supports_fd else fpath,\n                     dir_fd=None if os.supports_fd else dir_fd, **kwargs)\n    return fpath", "response": "touch a file in the cache"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nremoves a file or recursively removes a directory.", "response": "def delete(path, verbose=False):\n    \"\"\"\n    Removes a file or recursively removes a directory.\n    If a path does not exist, then this is does nothing.\n\n    Args:\n        path (PathLike): file or directory to remove\n        verbose (bool): if True prints what is being done\n\n    SeeAlso:\n        send2trash - A cross-platform Python package for sending files\n            to the trash instead of irreversibly deleting them.\n            https://github.com/hsoft/send2trash\n\n    Doctest:\n        >>> import ubelt as ub\n        >>> base = ub.ensure_app_cache_dir('ubelt', 'delete_test')\n        >>> dpath1 = ub.ensuredir(join(base, 'dir'))\n        >>> ub.ensuredir(join(base, 'dir', 'subdir'))\n        >>> ub.touch(join(base, 'dir', 'to_remove1.txt'))\n        >>> fpath1 = join(base, 'dir', 'subdir', 'to_remove3.txt')\n        >>> fpath2 = join(base, 'dir', 'subdir', 'to_remove2.txt')\n        >>> ub.touch(fpath1)\n        >>> ub.touch(fpath2)\n        >>> assert all(map(exists, (dpath1, fpath1, fpath2)))\n        >>> ub.delete(fpath1)\n        >>> assert all(map(exists, (dpath1, fpath2)))\n        >>> assert not exists(fpath1)\n        >>> ub.delete(dpath1)\n        >>> assert not any(map(exists, (dpath1, fpath1, fpath2)))\n\n    Doctest:\n        >>> import ubelt as ub\n        >>> dpath = ub.ensure_app_cache_dir('ubelt', 'delete_test2')\n        >>> dpath1 = ub.ensuredir(join(dpath, 'dir'))\n        >>> fpath1 = ub.touch(join(dpath1, 'to_remove.txt'))\n        >>> assert exists(fpath1)\n        >>> ub.delete(dpath)\n        >>> assert not exists(fpath1)\n    \"\"\"\n    if not os.path.exists(path):\n        # if the file does exists and is not a broken link\n        if os.path.islink(path):\n            if verbose:  # nocover\n                print('Deleting broken link=\"{}\"'.format(path))\n            os.unlink(path)\n        elif os.path.isdir(path):  # nocover\n            # Only on windows will a file be a directory and not exist\n            if verbose:\n                print('Deleting broken directory link=\"{}\"'.format(path))\n            os.rmdir(path)\n        elif os.path.isfile(path):  # nocover\n            # This is a windows only case\n            if verbose:\n                print('Deleting broken file link=\"{}\"'.format(path))\n            os.unlink(path)\n        else:\n            if verbose:  # nocover\n                print('Not deleting non-existant path=\"{}\"'.format(path))\n    else:\n        if os.path.islink(path):\n            if verbose:  # nocover\n                print('Deleting symbolic link=\"{}\"'.format(path))\n            os.unlink(path)\n        elif os.path.isfile(path):\n            if verbose:  # nocover\n                print('Deleting file=\"{}\"'.format(path))\n            os.unlink(path)\n        elif os.path.isdir(path):\n            if verbose:  # nocover\n                print('Deleting directory=\"{}\"'.format(path))\n            if sys.platform.startswith('win32'):  # nocover\n                # Workaround bug that prevents shutil from working if\n                # the directory contains junctions\n                from ubelt import _win32_links\n                _win32_links._win32_rmtree(path, verbose=verbose)\n            else:\n                import shutil\n                shutil.rmtree(path)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef repr2(data, **kwargs):\n    custom_extensions = kwargs.get('extensions', None)\n\n    _return_info = kwargs.get('_return_info', False)\n    kwargs['_root_info'] = _rectify_root_info(kwargs.get('_root_info', None))\n\n    outstr = None\n    _leaf_info = None\n\n    if custom_extensions:\n        func = custom_extensions.lookup(data)\n        if func is not None:\n            outstr = func(data, **kwargs)\n\n    if outstr is None:\n        if isinstance(data, dict):\n            outstr, _leaf_info = _format_dict(data, **kwargs)\n        elif isinstance(data, (list, tuple, set, frozenset)):\n            outstr, _leaf_info = _format_list(data, **kwargs)\n\n    if outstr is None:\n        # check any globally registered functions for special formatters\n        func = _FORMATTER_EXTENSIONS.lookup(data)\n        if func is not None:\n            outstr = func(data, **kwargs)\n        else:\n            outstr = _format_object(data, **kwargs)\n\n    if _return_info:\n        _leaf_info = _rectify_leaf_info(_leaf_info)\n        return outstr, _leaf_info\n    else:\n        return outstr", "response": "Return a pretty and easy - to - doctest string representation of the object."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _format_list(list_, **kwargs):\n    kwargs['_root_info'] = _rectify_root_info(kwargs.get('_root_info', None))\n    kwargs['_root_info']['depth'] += 1\n\n    newlines = kwargs.pop('nl', kwargs.pop('newlines', 1))\n    kwargs['nl'] = _rectify_countdown_or_bool(newlines)\n\n    nobraces = kwargs.pop('nobr', kwargs.pop('nobraces', False))\n\n    itemsep = kwargs.get('itemsep', ' ')\n\n    compact_brace = kwargs.get('cbr', kwargs.get('compact_brace', False))\n    # kwargs['cbr'] = _rectify_countdown_or_bool(compact_brace)\n\n    itemstrs, _leaf_info = _list_itemstrs(list_, **kwargs)\n    if len(itemstrs) == 0:\n        nobraces = False  # force braces to prevent empty output\n\n    is_tuple = isinstance(list_, tuple)\n    is_set = isinstance(list_, (set, frozenset,))\n    if nobraces:\n        lbr, rbr = '', ''\n    elif is_tuple:\n        lbr, rbr  = '(', ')'\n    elif is_set:\n        lbr, rbr  = '{', '}'\n    else:\n        lbr, rbr  = '[', ']'\n\n    # Doesn't actually put in trailing comma if on same line\n    trailing_sep = kwargs.get('trailsep', kwargs.get('trailing_sep', newlines > 0 and len(itemstrs)))\n\n    # The trailing separator is always needed for single item tuples\n    if is_tuple and len(list_) <= 1:\n        trailing_sep = True\n\n    if len(itemstrs) == 0:\n        newlines = False\n\n    retstr = _join_itemstrs(itemstrs, itemsep, newlines, _leaf_info, nobraces,\n                            trailing_sep, compact_brace, lbr, rbr)\n    return retstr, _leaf_info", "response": "Takes a list of items and returns a pretty printable string representation of the items in the tree."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nformats a dictionary into a pretty printable string representation.", "response": "def _format_dict(dict_, **kwargs):\n    \"\"\"\n    Makes a pretty printable / human-readable string representation of a\n    dictionary. In most cases this string could be evaled.\n\n    Args:\n        dict_ (dict):  a dictionary\n        **kwargs: si, stritems, strkeys, strvals, sk, sv, nl, newlines, nobr,\n                  nobraces, cbr, compact_brace, trailing_sep,\n                  explicit, itemsep, precision, kvsep, sort\n\n    Returns:\n        Tuple[str, Dict] : retstr, _leaf_info\n\n    Kwargs:\n        sort (None): if True, sorts ALL collections and subcollections,\n            note, collections with undefined orders (e.g. dicts, sets) are\n            sorted by default. (default = None)\n        nl (int): preferred alias for newline. can be a countdown variable\n            (default = None)\n        explicit (int): can be a countdown variable. if True, uses\n            dict(a=b) syntax instead of {'a': b}\n        nobr (bool): removes outer braces (default = False)\n    \"\"\"\n    kwargs['_root_info'] = _rectify_root_info(kwargs.get('_root_info', None))\n    kwargs['_root_info']['depth'] += 1\n\n    stritems = kwargs.pop('si', kwargs.pop('stritems', False))\n    if stritems:\n        kwargs['strkeys'] = True\n        kwargs['strvals'] = True\n\n    kwargs['strkeys'] = kwargs.pop('sk', kwargs.pop('strkeys', False))\n    kwargs['strvals'] = kwargs.pop('sv', kwargs.pop('strvals', False))\n\n    newlines = kwargs.pop('nl', kwargs.pop('newlines', True))\n    kwargs['nl'] = _rectify_countdown_or_bool(newlines)\n\n    nobraces = kwargs.pop('nobr', kwargs.pop('nobraces', False))\n\n    compact_brace = kwargs.get('cbr', kwargs.get('compact_brace', False))\n    # kwargs['cbr'] = _rectify_countdown_or_bool(compact_brace)\n\n    # Doesn't actually put in trailing comma if on same line\n    trailing_sep = kwargs.get('trailsep', kwargs.get('trailing_sep', newlines > 0))\n    explicit = kwargs.get('explicit', False)\n    itemsep = kwargs.get('itemsep', ' ')\n\n    if len(dict_) == 0:\n        retstr = 'dict()' if explicit else '{}'\n        _leaf_info = None\n    else:\n        itemstrs, _leaf_info = _dict_itemstrs(dict_, **kwargs)\n        if nobraces:\n            lbr, rbr = '', ''\n        elif explicit:\n            lbr, rbr = 'dict(', ')'\n        else:\n            lbr, rbr = '{', '}'\n        retstr = _join_itemstrs(itemstrs, itemsep, newlines, _leaf_info, nobraces,\n                                trailing_sep, compact_brace, lbr, rbr)\n    return retstr, _leaf_info"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\njoin string -ified items with separators newlines and container - braces.", "response": "def _join_itemstrs(itemstrs, itemsep, newlines, _leaf_info, nobraces,\n                   trailing_sep, compact_brace, lbr, rbr):\n    \"\"\"\n    Joins string-ified items with separators newlines and container-braces.\n    \"\"\"\n    # positive newlines means start counting from the root\n    use_newline = newlines > 0\n\n    # negative countdown values mean start counting from the leafs\n    # if compact_brace < 0:\n    #     compact_brace = (-compact_brace) >= _leaf_info['max_height']\n    if newlines < 0:\n        use_newline = (-newlines) < _leaf_info['max_height']\n\n    if use_newline:\n        sep = ',\\n'\n        if nobraces:\n            body_str = sep.join(itemstrs)\n            if trailing_sep and len(itemstrs) > 0:\n                body_str += ','\n            retstr = body_str\n        else:\n            if compact_brace:\n                # Why must we modify the indentation below and not here?\n                # prefix = ''\n                # rest = [ub.indent(s, prefix) for s in itemstrs[1:]]\n                # indented = itemstrs[0:1] + rest\n                indented = itemstrs\n            else:\n                import ubelt as ub\n                prefix = ' ' * 4\n                indented = [ub.indent(s, prefix) for s in itemstrs]\n\n            body_str = sep.join(indented)\n            if trailing_sep and len(itemstrs) > 0:\n                body_str += ','\n            if compact_brace:\n                # Why can we modify the indentation here but not above?\n                braced_body_str = (lbr + body_str.replace('\\n', '\\n ') + rbr)\n            else:\n                braced_body_str = (lbr + '\\n' + body_str + '\\n' + rbr)\n            retstr = braced_body_str\n    else:\n        sep = ',' + itemsep\n        body_str = sep.join(itemstrs)\n        if trailing_sep and len(itemstrs) > 0:\n            body_str += ','\n        retstr  = (lbr + body_str +  rbr)\n    return retstr"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncreating a string representation for each item in a dict.", "response": "def _dict_itemstrs(dict_, **kwargs):\n    \"\"\"\n    Create a string representation for each item in a dict.\n\n    Example:\n        >>> from ubelt.util_format import *\n        >>> dict_ =  {'b': .1, 'l': 'st', 'g': 1.0, 's': 10, 'm': 0.9, 'w': .5}\n        >>> kwargs = {'strkeys': True}\n        >>> itemstrs, _ = _dict_itemstrs(dict_, **kwargs)\n        >>> char_order = [p[0] for p in itemstrs]\n        >>> assert char_order == ['b', 'g', 'l', 'm', 's', 'w']\n    \"\"\"\n    import ubelt as ub\n    explicit = kwargs.get('explicit', False)\n    kwargs['explicit'] = _rectify_countdown_or_bool(explicit)\n    precision = kwargs.get('precision', None)\n    kvsep = kwargs.get('kvsep', ': ')\n    if explicit:\n        kvsep = '='\n\n    def make_item_str(key, val):\n        if explicit or kwargs.get('strkeys', False):\n            key_str = six.text_type(key)\n        else:\n            key_str = repr2(key, precision=precision, newlines=0)\n\n        prefix = key_str + kvsep\n        kwargs['_return_info'] = True\n        val_str, _leaf_info = repr2(val, **kwargs)\n\n        # If the first line does not end with an open nest char\n        # (e.g. for ndarrays), otherwise we need to worry about\n        # residual indentation.\n        pos = val_str.find('\\n')\n        first_line = val_str if pos == -1 else val_str[:pos]\n\n        compact_brace = kwargs.get('cbr', kwargs.get('compact_brace', False))\n\n        if compact_brace or not first_line.rstrip().endswith(tuple('([{<')):\n            rest = '' if pos == -1 else val_str[pos:]\n            val_str = first_line.lstrip() + rest\n            if '\\n' in prefix:\n                # Fix issue with keys that span new lines\n                item_str = prefix + val_str\n            else:\n                item_str = ub.hzcat([prefix, val_str])\n        else:\n            item_str = prefix + val_str\n        return item_str, _leaf_info\n\n    items = list(six.iteritems(dict_))\n    _tups = [make_item_str(key, val) for (key, val) in items]\n    itemstrs = [t[0] for t in _tups]\n    max_height = max([t[1]['max_height'] for t in _tups]) if _tups else 0\n    _leaf_info = {\n        'max_height': max_height + 1,\n    }\n\n    sort = kwargs.get('sort', None)\n    if sort is None:\n        # Force ordering on unordered dicts\n        sort = True\n    if isinstance(dict_, collections.OrderedDict):\n        # never sort ordered dicts; they are perfect just the way they are!\n        sort = False\n    if sort:\n        itemstrs = _sort_itemstrs(items, itemstrs)\n    return itemstrs, _leaf_info"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _list_itemstrs(list_, **kwargs):\n    items = list(list_)\n    kwargs['_return_info'] = True\n    _tups = [repr2(item, **kwargs) for item in items]\n    itemstrs = [t[0] for t in _tups]\n    max_height = max([t[1]['max_height'] for t in _tups]) if _tups else 0\n    _leaf_info = {\n        'max_height': max_height + 1,\n    }\n\n    sort = kwargs.get('sort', None)\n    if sort is None:\n        # Force orderings on sets.\n        sort = isinstance(list_, (set, frozenset))\n    if sort:\n        itemstrs = _sort_itemstrs(items, itemstrs)\n    return itemstrs, _leaf_info", "response": "Create a string representation for each item in a list."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _rectify_countdown_or_bool(count_or_bool):\n    if count_or_bool is True or count_or_bool is False:\n        count_or_bool_ = count_or_bool\n    elif isinstance(count_or_bool, int):\n        if count_or_bool == 0:\n            return 0\n        elif count_or_bool > 0:\n            count_or_bool_ = count_or_bool - 1\n        else:\n            # We dont countup negatives anymore\n            count_or_bool_ = count_or_bool\n    else:\n        count_or_bool_ = False\n    return count_or_bool_", "response": "Returns a list of the countdowns of the current or negative ones."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef register(self, type):\n        def _decorator(func):\n            if isinstance(type, tuple):\n                for t in type:\n                    self.func_registry[t] = func\n            else:\n                self.func_registry[type] = func\n            return func\n        return _decorator", "response": "Registers a custom formatting function with ub. repr2\n       "}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning an appropriate function to format data if one has been registered.", "response": "def lookup(self, data):\n        \"\"\"\n        Returns an appropriate function to format `data` if one has been\n        registered.\n        \"\"\"\n        for func in self.lazy_init:\n            func()\n\n        for type, func in self.func_registry.items():\n            if isinstance(data, type):\n                return func"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _register_numpy_extensions(self):\n        import numpy as np\n        @self.register(np.ndarray)\n        def format_ndarray(data, **kwargs):\n            import re\n            strvals = kwargs.get('sv', kwargs.get('strvals', False))\n            itemsep = kwargs.get('itemsep', ' ')\n            precision = kwargs.get('precision', None)\n            suppress_small = kwargs.get('supress_small', None)\n            max_line_width = kwargs.get('max_line_width', None)\n            with_dtype = kwargs.get('with_dtype', kwargs.get('dtype', not strvals))\n            newlines = kwargs.pop('nl', kwargs.pop('newlines', 1))\n\n            # if with_dtype and strvals:\n            #     raise ValueError('cannot format with strvals and dtype')\n\n            separator = ',' + itemsep\n\n            if strvals:\n                prefix = ''\n                suffix = ''\n            else:\n                modname = type(data).__module__\n                # substitute shorthand for numpy module names\n                np_nice = 'np'\n                modname = re.sub('\\\\bnumpy\\\\b', np_nice, modname)\n                modname = re.sub('\\\\bma.core\\\\b', 'ma', modname)\n\n                class_name = type(data).__name__\n                if class_name == 'ndarray':\n                    class_name = 'array'\n\n                prefix = modname + '.' + class_name + '('\n\n                if with_dtype:\n                    dtype_repr = data.dtype.name\n                    # dtype_repr = np.core.arrayprint.dtype_short_repr(data.dtype)\n                    suffix = ',{}dtype={}.{})'.format(itemsep, np_nice, dtype_repr)\n                else:\n                    suffix = ')'\n\n            if not strvals and data.size == 0 and data.shape != (0,):\n                # Special case for displaying empty data\n                prefix = modname + '.empty('\n                body = repr(tuple(map(int, data.shape)))\n            else:\n                body = np.array2string(data, precision=precision,\n                                       separator=separator,\n                                       suppress_small=suppress_small,\n                                       prefix=prefix,\n                                       max_line_width=max_line_width)\n            if not newlines:\n                # remove newlines if we need to\n                body = re.sub('\\n *', '', body)\n            formatted = prefix + body + suffix\n            return formatted\n\n        # Hack, make sure we also register numpy floats\n        self.register(np.float32)(self.func_registry[float])", "response": "Registers numpy extensions for the base class."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _rectify_hasher(hasher):\n    if xxhash is not None:  # pragma: nobranch\n        if hasher in {'xxh32', 'xx32', 'xxhash'}:\n            return xxhash.xxh32\n        if hasher in {'xxh64', 'xx64'}:\n            return xxhash.xxh64\n\n    if hasher is NoParam or hasher == 'default':\n        hasher = DEFAULT_HASHER\n    elif isinstance(hasher, six.string_types):\n        if hasher not in hashlib.algorithms_available:\n            raise KeyError('unknown hasher: {}'.format(hasher))\n        else:\n            hasher = getattr(hashlib, hasher)\n    elif isinstance(hasher, HASH):\n        # by default the result of this function is a class we will make an\n        # instance of, if we already have an instance, wrap it in a callable\n        # so the external syntax does not need to change.\n        return lambda: hasher\n    return hasher", "response": "Convert a string - based key into a hasher class."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _rectify_base(base):\n    if base is NoParam or base == 'default':\n        return DEFAULT_ALPHABET\n    elif base in [26, 'abc', 'alpha']:\n        return _ALPHABET_26\n    elif base in [16, 'hex']:\n        return _ALPHABET_16\n    elif base in [10, 'dec']:\n        return _ALPHABET_10\n    else:\n        if not isinstance(base, (list, tuple)):\n            raise TypeError(\n                'Argument `base` must be a key, list, or tuple; not {}'.format(\n                    type(base)))\n        return base", "response": "Returns the full list representation of the base key."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _hashable_sequence(data, types=True):\n    hasher = _HashTracer()\n    _update_hasher(hasher, data, types=types)\n    return hasher.sequence", "response": "r Returns the sequence of bytes that would be hashed by hash_data\n\n   "}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _update_hasher(hasher, data, types=True):\n    # Determine if the data should be hashed directly or iterated through\n    if isinstance(data, (tuple, list, zip)):\n        needs_iteration = True\n    else:\n        needs_iteration = any(check(data) for check in\n                              _HASHABLE_EXTENSIONS.iterable_checks)\n\n    if needs_iteration:\n        # Denote that we are hashing over an iterable\n        # Multiple structure bytes makes it harder accidently make conflicts\n        SEP = b'_,_'\n        ITER_PREFIX = b'_[_'\n        ITER_SUFFIX = b'_]_'\n\n        iter_ = iter(data)\n        hasher.update(ITER_PREFIX)\n        # first, try to nest quickly without recursive calls\n        # (this works if all data in the sequence is a non-iterable)\n        try:\n            for item in iter_:\n                prefix, hashable = _convert_to_hashable(item, types)\n                binary_data = prefix + hashable + SEP\n                hasher.update(binary_data)\n        except TypeError:\n            # need to use recursive calls\n            # Update based on current item\n            _update_hasher(hasher, item, types)\n            for item in iter_:\n                # Ensure the items have a spacer between them\n                _update_hasher(hasher, item, types)\n                hasher.update(SEP)\n        hasher.update(ITER_SUFFIX)\n    else:\n        prefix, hashable = _convert_to_hashable(data, types)\n        binary_data = prefix + hashable\n        hasher.update(binary_data)", "response": "Updates the hasher with the data in the order of the data."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef hash_data(data, hasher=NoParam, base=NoParam, types=False,\n              hashlen=NoParam, convert=False):\n    \"\"\"\n    Get a unique hash depending on the state of the data.\n\n    Args:\n        data (object):\n            Any sort of loosely organized data\n\n        hasher (str or HASHER):\n            Hash algorithm from hashlib, defaults to `sha512`.\n\n        base (str or List[str]):\n            Shorthand key or a list of symbols.  Valid keys are: 'abc', 'hex',\n            and 'dec'. Defaults to 'hex'.\n\n        types (bool):\n            If True data types are included in the hash, otherwise only the raw\n            data is hashed. Defaults to False.\n\n        hashlen (int):\n            Maximum number of symbols in the returned hash. If not specified,\n            all are returned.  DEPRECATED. Use slice syntax instead.\n\n        convert (bool, optional, default=True):\n            if True, try and convert the data to json an the json is hashed\n            instead. This can improve runtime in some instances, however the\n            hash may differ from the case where convert=False.\n\n    Notes:\n        alphabet26 is a pretty nice base, I recommend it.\n        However we default to hex because it is standard.\n        This means the output of hashdata with base=sha1 will be the same as\n        the output of `sha1sum`.\n\n    Returns:\n        str: text -  hash string\n\n    Example:\n        >>> import ubelt as ub\n        >>> print(ub.hash_data([1, 2, (3, '4')], convert=False))\n        60b758587f599663931057e6ebdf185a...\n        >>> print(ub.hash_data([1, 2, (3, '4')], base='abc',  hasher='sha512')[:32])\n        hsrgqvfiuxvvhcdnypivhhthmrolkzej\n    \"\"\"\n    if convert and isinstance(data, six.string_types):  # nocover\n        try:\n            data = json.dumps(data)\n        except TypeError as ex:\n            # import warnings\n            # warnings.warn('Unable to encode input as json due to: {!r}'.format(ex))\n            pass\n\n    base = _rectify_base(base)\n    hashlen = _rectify_hashlen(hashlen)\n    hasher = _rectify_hasher(hasher)()\n    # Feed the data into the hasher\n    _update_hasher(hasher, data, types=types)\n    # Get the hashed representation\n    text = _digest_hasher(hasher, hashlen, base)\n    return text", "response": "Return a unique hash of the data."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef hash_file(fpath, blocksize=65536, stride=1, hasher=NoParam,\n              hashlen=NoParam, base=NoParam):\n    \"\"\"\n    Hashes the data in a file on disk.\n\n    Args:\n        fpath (PathLike):  file path string\n\n        blocksize (int): 2 ** 16. Affects speed of reading file\n\n        stride (int): strides > 1 skip data to hash, useful for faster\n                      hashing, but less accurate, also makes hash dependant on\n                      blocksize.\n\n        hasher (HASH): hash algorithm from hashlib, defaults to `sha512`.\n\n        hashlen (int): maximum number of symbols in the returned hash. If\n            not specified, all are returned.\n\n        base (list, str): list of symbols or shorthand key. Valid keys are\n            'abc', 'hex', and 'dec'. Defaults to 'hex'.\n\n    Notes:\n        For better hashes keep stride = 1\n        For faster hashes set stride > 1\n        blocksize matters when stride > 1\n\n    References:\n        http://stackoverflow.com/questions/3431825/md5-checksum-of-a-file\n        http://stackoverflow.com/questions/5001893/when-to-use-sha-1-vs-sha-2\n\n    Example:\n        >>> import ubelt as ub\n        >>> from os.path import join\n        >>> fpath = join(ub.ensure_app_cache_dir('ubelt'), 'tmp.txt')\n        >>> ub.writeto(fpath, 'foobar')\n        >>> print(ub.hash_file(fpath, hasher='sha1', base='hex'))\n        8843d7f92416211de9ebb963ff4ce28125932878\n\n    Example:\n        >>> import ubelt as ub\n        >>> from os.path import join\n        >>> fpath = ub.touch(join(ub.ensure_app_cache_dir('ubelt'), 'empty_file'))\n        >>> # Test that the output is the same as sha1sum\n        >>> if ub.find_exe('sha1sum'):\n        >>>     want = ub.cmd(['sha1sum', fpath], verbose=2)['out'].split(' ')[0]\n        >>>     got = ub.hash_file(fpath, hasher='sha1')\n        >>>     print('want = {!r}'.format(want))\n        >>>     print('got = {!r}'.format(got))\n        >>>     assert want.endswith(got)\n        >>> # Do the same for sha512 sum and md5sum\n        >>> if ub.find_exe('sha512sum'):\n        >>>     want = ub.cmd(['sha512sum', fpath], verbose=2)['out'].split(' ')[0]\n        >>>     got = ub.hash_file(fpath, hasher='sha512')\n        >>>     print('want = {!r}'.format(want))\n        >>>     print('got = {!r}'.format(got))\n        >>>     assert want.endswith(got)\n        >>> if ub.find_exe('md5sum'):\n        >>>     want = ub.cmd(['md5sum', fpath], verbose=2)['out'].split(' ')[0]\n        >>>     got = ub.hash_file(fpath, hasher='md5')\n        >>>     print('want = {!r}'.format(want))\n        >>>     print('got = {!r}'.format(got))\n        >>>     assert want.endswith(got)\n    \"\"\"\n    base = _rectify_base(base)\n    hashlen = _rectify_hashlen(hashlen)\n    hasher = _rectify_hasher(hasher)()\n    with open(fpath, 'rb') as file:\n        buf = file.read(blocksize)\n        if stride > 1:\n            # skip blocks when stride is greater than 1\n            while len(buf) > 0:\n                hasher.update(buf)\n                file.seek(blocksize * (stride - 1), 1)\n                buf = file.read(blocksize)\n        else:\n            # otherwise hash the entire file\n            while len(buf) > 0:\n                hasher.update(buf)\n                buf = file.read(blocksize)\n    # Get the hashed representation\n    text = _digest_hasher(hasher, hashlen, base)\n    return text", "response": "Hashes the contents of a file on disk."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nregistering a function to generate a hash for the appropriate base - level data types.", "response": "def register(self, hash_types):\n        \"\"\"\n        Registers a function to generate a hash for data of the appropriate\n        types. This can be used to register custom classes. Internally this is\n        used to define how to hash non-builtin objects like ndarrays and uuids.\n\n        The registered function should return a tuple of bytes. First a small\n        prefix hinting at the data type, and second the raw bytes that can be\n        hashed.\n\n        Args:\n            hash_types (class or tuple of classes):\n\n        Returns:\n            func: closure to be used as the decorator\n\n        Example:\n            >>> # xdoctest: +SKIP\n            >>> # Skip this doctest because we dont want tests to modify\n            >>> # the global state.\n            >>> import ubelt as ub\n            >>> import pytest\n            >>> class MyType(object):\n            ...     def __init__(self, id):\n            ...         self.id = id\n            >>> data = MyType(1)\n            >>> # Custom types wont work with ub.hash_data by default\n            >>> with pytest.raises(TypeError):\n            ...     ub.hash_data(data)\n            >>> # You can register your functions with ubelt's internal\n            >>> # hashable_extension registery.\n            >>> @ub.util_hash._HASHABLE_EXTENSIONS.register(MyType)\n            >>> def hash_my_type(data):\n            ...     return b'mytype', six.b(ub.hash_data(data.id))\n            >>> # TODO: allow hash_data to take an new instance of\n            >>> # HashableExtensions, so we dont have to modify the global\n            >>> # ubelt state when we run tests.\n            >>> my_instance = MyType(1)\n            >>> ub.hash_data(my_instance)\n        \"\"\"\n        # ensure iterable\n        if not isinstance(hash_types, (list, tuple)):\n            hash_types = [hash_types]\n        def _decor_closure(hash_func):\n            for hash_type in hash_types:\n                key = (hash_type.__module__, hash_type.__name__)\n                self.keyed_extensions[key] = (hash_type, hash_func)\n            return hash_func\n        return _decor_closure"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef lookup(self, data):\n        # Maybe try using functools.singledispatch instead?\n        # First try O(1) lookup\n        query_hash_type = data.__class__\n        key = (query_hash_type.__module__, query_hash_type.__name__)\n        try:\n            hash_type, hash_func = self.keyed_extensions[key]\n        except KeyError:\n            raise TypeError('No registered hash func for hashable type=%r' % (\n                    query_hash_type))\n        return hash_func", "response": "Returns an appropriate function to hash data if one has been registered."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _register_numpy_extensions(self):\n        # system checks\n        import numpy as np\n        numpy_floating_types = (np.float16, np.float32, np.float64)\n        if hasattr(np, 'float128'):  # nocover\n            numpy_floating_types = numpy_floating_types + (np.float128,)\n\n        @self.add_iterable_check\n        def is_object_ndarray(data):\n            # ndarrays of objects cannot be hashed directly.\n            return isinstance(data, np.ndarray) and data.dtype.kind == 'O'\n\n        @self.register(np.ndarray)\n        def hash_numpy_array(data):\n            \"\"\"\n            Example:\n                >>> import ubelt as ub\n                >>> if not ub.modname_to_modpath('numpy'):\n                ...     raise pytest.skip()\n                >>> import numpy as np\n                >>> data_f32 = np.zeros((3, 3, 3), dtype=np.float64)\n                >>> data_i64 = np.zeros((3, 3, 3), dtype=np.int64)\n                >>> data_i32 = np.zeros((3, 3, 3), dtype=np.int32)\n                >>> hash_f64 = _hashable_sequence(data_f32, types=True)\n                >>> hash_i64 = _hashable_sequence(data_i64, types=True)\n                >>> hash_i32 = _hashable_sequence(data_i64, types=True)\n                >>> assert hash_i64 != hash_f64\n                >>> assert hash_i64 != hash_i32\n            \"\"\"\n            if data.dtype.kind == 'O':\n                msg = 'directly hashing ndarrays with dtype=object is unstable'\n                raise TypeError(msg)\n            else:\n                # tobytes() views the array in 1D (via ravel())\n                # encode the shape as well\n                header = b''.join(_hashable_sequence((len(data.shape), data.shape)))\n                dtype = b''.join(_hashable_sequence(data.dtype.descr))\n                hashable = header + dtype + data.tobytes()\n            prefix = b'NDARR'\n            return prefix, hashable\n\n        @self.register((np.int64, np.int32, np.int16, np.int8) +\n                       (np.uint64, np.uint32, np.uint16, np.uint8))\n        def _hash_numpy_int(data):\n            return _convert_to_hashable(int(data))\n\n        @self.register(numpy_floating_types)\n        def _hash_numpy_float(data):\n            return _convert_to_hashable(float(data))\n\n        @self.register(np.random.RandomState)\n        def _hash_numpy_random_state(data):\n            \"\"\"\n            Example:\n                >>> import ubelt as ub\n                >>> if not ub.modname_to_modpath('numpy'):\n                ...     raise pytest.skip()\n                >>> import numpy as np\n                >>> rng = np.random.RandomState(0)\n                >>> _hashable_sequence(rng, types=True)\n            \"\"\"\n            hashable = b''.join(_hashable_sequence(data.get_state()))\n            prefix = b'RNG'\n            return prefix, hashable", "response": "Numpy extensions are builtin"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _register_builtin_class_extensions(self):\n        @self.register(uuid.UUID)\n        def _hash_uuid(data):\n            hashable = data.bytes\n            prefix = b'UUID'\n            return prefix, hashable\n\n        @self.register(OrderedDict)\n        def _hash_ordered_dict(data):\n            \"\"\"\n            Note, we should not be hashing dicts because they are unordered\n            \"\"\"\n            hashable = b''.join(_hashable_sequence(list(data.items())))\n            prefix = b'ODICT'\n            return prefix, hashable", "response": "Register builtin class extensions for a selection of classes included in the current module."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\niterates over lines in a TextIO stream until an EOF is encountered.", "response": "def _textio_iterlines(stream):\n    \"\"\"\n    Iterates over lines in a TextIO stream until an EOF is encountered.\n    This is the iterator version of stream.readlines()\n    \"\"\"\n    line = stream.readline()\n    while line != '':\n        yield line\n        line = stream.readline()"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nread output from a process in a separate thread", "response": "def _proc_async_iter_stream(proc, stream, buffersize=1):\n    \"\"\"\n    Reads output from a process in a separate thread\n    \"\"\"\n    from six.moves import queue\n    from threading import Thread\n    def enqueue_output(proc, stream, stream_queue):\n        while proc.poll() is None:\n            line = stream.readline()\n            # print('ENQUEUE LIVE {!r} {!r}'.format(stream, line))\n            stream_queue.put(line)\n\n        for line in _textio_iterlines(stream):\n            # print('ENQUEUE FINAL {!r} {!r}'.format(stream, line))\n            stream_queue.put(line)\n\n        # print(\"STREAM IS DONE {!r}\".format(stream))\n        stream_queue.put(None)  # signal that the stream is finished\n        # stream.close()\n    stream_queue = queue.Queue(maxsize=buffersize)\n    _thread = Thread(target=enqueue_output, args=(proc, stream, stream_queue))\n    _thread.daemon = True  # thread dies with the program\n    _thread.start()\n    return stream_queue"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\niterating over the output of a process line by line by linebyline.", "response": "def _proc_iteroutput_thread(proc):\n    \"\"\"\n    Iterates over output from a process line by line\n\n    Note:\n        WARNING. Current implementation might have bugs with other threads.\n        This behavior was seen when using earlier versions of tqdm. I'm not\n        sure if this was our bug or tqdm's. Newer versions of tqdm fix this,\n        but I cannot guarantee that there isn't an issue on our end.\n\n    Yields:\n        Tuple[str, str]: oline, eline: stdout and stderr line\n\n    References:\n        https://stackoverflow.com/questions/375427/non-blocking-read-subproc\n    \"\"\"\n    from six.moves import queue\n\n    # Create threads that read stdout / stderr and queue up the output\n    stdout_queue = _proc_async_iter_stream(proc, proc.stdout)\n    stderr_queue = _proc_async_iter_stream(proc, proc.stderr)\n\n    stdout_live = True\n    stderr_live = True\n\n    # read from the output asynchronously until\n    while stdout_live or stderr_live:\n        if stdout_live:  # pragma: nobranch\n            try:\n                oline = stdout_queue.get_nowait()\n                stdout_live = oline is not None\n            except queue.Empty:\n                oline = None\n        if stderr_live:\n            try:\n                eline = stderr_queue.get_nowait()\n                stderr_live = eline is not None\n            except queue.Empty:\n                eline = None\n        if oline is not None or eline is not None:\n            yield oline, eline"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _proc_iteroutput_select(proc):\n    from six.moves import zip_longest\n    # Read output while the external program is running\n    while proc.poll() is None:\n        reads = [proc.stdout.fileno(), proc.stderr.fileno()]\n        ret = select.select(reads, [], [])\n        oline = eline = None\n        for fd in ret[0]:\n            if fd == proc.stdout.fileno():\n                oline = proc.stdout.readline()\n            if fd == proc.stderr.fileno():\n                eline = proc.stderr.readline()\n        yield oline, eline\n\n    # Grab any remaining data in stdout and stderr after the process finishes\n    oline_iter = _textio_iterlines(proc.stdout)\n    eline_iter = _textio_iterlines(proc.stderr)\n    for oline, eline in zip_longest(oline_iter, eline_iter):\n        yield oline, eline", "response": "Yields the output of a process line by line\n    UNIX only."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _tee_output(make_proc, stdout=None, stderr=None, backend='auto'):\n    logged_out = []\n    logged_err = []\n    if backend == 'auto':\n        # backend = 'select' if POSIX else 'thread'\n        backend = 'thread'\n\n    if backend == 'select':\n        if not POSIX:  # nocover\n            raise NotImplementedError('select is only available on posix')\n        # the select-based version is stable, but slow\n        _proc_iteroutput = _proc_iteroutput_select\n    elif backend == 'thread':\n        # the thread version is fast, but might run into issues.\n        _proc_iteroutput = _proc_iteroutput_thread\n    else:\n        raise ValueError('backend must be select, thread, or auto')\n\n    proc = make_proc()\n    for oline, eline in _proc_iteroutput(proc):\n        if oline:\n            if stdout:  # pragma: nobranch\n                stdout.write(oline)\n                stdout.flush()\n            logged_out.append(oline)\n        if eline:\n            if stderr:  # pragma: nobranch\n                stderr.write(eline)\n                stderr.flush()\n            logged_err.append(eline)\n    return proc, logged_out, logged_err", "response": "A helper function that returns a process that returns stdout and stderr from a process that yields a list of log entries."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cmd(command, shell=False, detach=False, verbose=0, tee=None, cwd=None,\n        env=None, tee_backend='auto', verbout=None, **kwargs):\n    \"\"\"\n    Executes a command in a subprocess.\n\n    The advantage of this wrapper around subprocess is that\n    (1) you control if the subprocess prints to stdout,\n    (2) the text written to stdout and stderr is returned for parsing,\n    (3) cross platform behavior that lets you specify the command as a string\n    or tuple regardless of whether or not shell=True.\n    (4) ability to detach, return the process object and allow the process to\n    run in the background (eventually we may return a Future object instead).\n\n    Args:\n        command (str or Sequence): bash-like command string or tuple of\n            executable and args\n\n        shell (bool): if True, process is run in shell, defaults to False.\n\n        detach (bool): if True, process is detached and run in background,\n            defaults to False.\n\n        verbose (int): verbosity mode. Can be 0, 1, 2, or 3. Defaults to 0.\n\n        tee (bool, optional): if True, simultaneously writes to stdout while\n            capturing output from the command. If not specified, defaults to\n            True if verbose > 0.  If detech is True, then this argument is\n            ignored.\n\n        cwd (PathLike, optional): path to run command\n\n        env (str, optional): environment passed to Popen\n\n        tee_backend (str, optional): backend for tee output.\n            Valid choices are: \"auto\", \"select\" (POSIX only), and \"thread\".\n\n        **kwargs: only used to support deprecated arguments\n\n    Returns:\n        dict: info - information about command status.\n            if detach is False `info` contains captured standard out,\n            standard error, and the return code\n            if detach is False `info` contains a reference to the process.\n\n    Notes:\n        Inputs can either be text or tuple based. On UNIX we ensure conversion\n        to text if shell=True, and to tuple if shell=False. On windows, the\n        input is always text based.  See [3] for a potential cross-platform\n        shlex solution for windows.\n\n    CommandLine:\n        python -m ubelt.util_cmd cmd\n        python -c \"import ubelt as ub; ub.cmd('ping localhost -c 2', verbose=2)\"\n\n    References:\n        [1] https://stackoverflow.com/questions/11495783/redirect-subprocess-stderr-to-stdout\n        [2] https://stackoverflow.com/questions/7729336/how-can-i-print-and-display-subprocess-stdout-and-stderr-output-without-distorti\n        [3] https://stackoverflow.com/questions/33560364/python-windows-parsing-command-lines-with-shlex\n\n    Example:\n        >>> info = cmd(('echo', 'simple cmdline interface'), verbose=1)\n        simple cmdline interface\n        >>> assert info['ret'] == 0\n        >>> assert info['out'].strip() == 'simple cmdline interface'\n        >>> assert info['err'].strip() == ''\n\n    Doctest:\n        >>> info = cmd('echo str noshell', verbose=0)\n        >>> assert info['out'].strip() == 'str noshell'\n\n    Doctest:\n        >>> # windows echo will output extra single quotes\n        >>> info = cmd(('echo', 'tuple noshell'), verbose=0)\n        >>> assert info['out'].strip().strip(\"'\") == 'tuple noshell'\n\n    Doctest:\n        >>> # Note this command is formatted to work on win32 and unix\n        >>> info = cmd('echo str&&echo shell', verbose=0, shell=True)\n        >>> assert info['out'].strip() == 'str' + chr(10) + 'shell'\n\n    Doctest:\n        >>> info = cmd(('echo', 'tuple shell'), verbose=0, shell=True)\n        >>> assert info['out'].strip().strip(\"'\") == 'tuple shell'\n\n    Doctest:\n        >>> import ubelt as ub\n        >>> from os.path import join, exists\n        >>> fpath1 = join(ub.get_app_cache_dir('ubelt'), 'cmdout1.txt')\n        >>> fpath2 = join(ub.get_app_cache_dir('ubelt'), 'cmdout2.txt')\n        >>> ub.delete(fpath1)\n        >>> ub.delete(fpath2)\n        >>> info1 = ub.cmd(('touch', fpath1), detach=True)\n        >>> info2 = ub.cmd('echo writing2 > ' + fpath2, shell=True, detach=True)\n        >>> while not exists(fpath1):\n        ...     pass\n        >>> while not exists(fpath2):\n        ...     pass\n        >>> assert ub.readfrom(fpath1) == ''\n        >>> assert ub.readfrom(fpath2).strip() == 'writing2'\n        >>> info1['proc'].wait()\n        >>> info2['proc'].wait()\n    \"\"\"\n    if kwargs:  # nocover\n        if 'verbout' in kwargs:\n            warnings.warn(\n                '`verbout` is deprecated and will be removed. '\n                'Use `tee` instead', DeprecationWarning)\n            tee = kwargs.pop('verbout')\n\n        if 'detatch' in kwargs:\n            warnings.warn(\n                '`detatch` is deprecated (misspelled) and will be removed. '\n                'Use `detach` instead', DeprecationWarning)\n            detach = kwargs.pop('detatch')\n\n        if kwargs:\n            raise ValueError('Unknown kwargs: {}'.format(list(kwargs.keys)))\n\n    # Determine if command is specified as text or a tuple\n    if isinstance(command, six.string_types):\n        command_text = command\n        command_tup = None\n    else:\n        import pipes\n        command_tup = command\n        command_text = ' '.join(list(map(pipes.quote, command_tup)))\n\n    if shell or sys.platform.startswith('win32'):\n        # When shell=True, args is sent to the shell (e.g. bin/sh) as text\n        args = command_text\n    else:\n        # When shell=False, args is a list of executable and arguments\n        if command_tup is None:\n            # parse this out of the string\n            # NOTE: perhaps use the solution from [3] here?\n            import shlex\n            command_tup = shlex.split(command_text)\n            # command_tup = shlex.split(command_text, posix=not WIN32)\n        args = command_tup\n\n    if tee is None:\n        tee = verbose > 0\n    if verbose > 1:\n        import os\n        import platform\n        import getpass\n        from ubelt import util_path\n        if verbose > 2:\n            try:\n                print('\u250c\u2500\u2500\u2500 START CMD \u2500\u2500\u2500')\n            except Exception:  # nocover\n                print('+=== START CMD ===')\n        cwd_ = os.getcwd() if cwd is None else cwd\n        compname = platform.node()\n        username = getpass.getuser()\n        cwd_ = util_path.compressuser(cwd_)\n        ps1 = '[ubelt.cmd] {}@{}:{}$ '.format(username, compname, cwd_)\n        print(ps1 + command_text)\n\n    # Create a new process to execute the command\n    def make_proc():\n        # delay the creation of the process until we validate all args\n        import subprocess\n        proc = subprocess.Popen(args, stdout=subprocess.PIPE,\n                                stderr=subprocess.PIPE, shell=shell,\n                                universal_newlines=True, cwd=cwd, env=env)\n        return proc\n\n    if detach:\n        info = {'proc': make_proc(), 'command': command_text}\n        if verbose > 0:  # nocover\n            print('...detaching')\n    else:\n        if tee:\n            # we need to tee output and start threads if tee is False?\n            stdout, stderr = sys.stdout, sys.stderr\n            proc, logged_out, logged_err = _tee_output(make_proc, stdout, stderr,\n                                                       backend=tee_backend)\n\n            try:\n                out = ''.join(logged_out)\n            except UnicodeDecodeError:  # nocover\n                out = '\\n'.join(_.decode('utf-8') for _ in logged_out)\n            try:\n                err = ''.join(logged_err)\n            except UnicodeDecodeError:  # nocover\n                err = '\\n'.join(_.decode('utf-8') for _ in logged_err)\n            (out_, err_) = proc.communicate()\n        else:\n            proc = make_proc()\n            (out, err) = proc.communicate()\n        # calling wait means that the process will terminate and it is safe to\n        # return a reference to the process object.\n        ret = proc.wait()\n        info = {\n            'out': out,\n            'err': err,\n            'ret': ret,\n            'proc': proc,\n            'cwd': cwd,\n            'command': command_text\n        }\n        if verbose > 2:\n            # https://en.wikipedia.org/wiki/Box-drawing_character\n            try:\n                print('\u2514\u2500\u2500\u2500 END CMD \u2500\u2500\u2500')\n            except Exception:  # nocover\n                print('L___ END CMD ___')\n    return info", "response": "Executes a command in a subprocess and returns a dictionary containing the output of the command."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nmake an ISO 8601 timestamp", "response": "def timestamp(method='iso8601'):\n    \"\"\"\n    make an iso8601 timestamp\n\n    Args:\n        method (str): type of timestamp\n\n    Example:\n        >>> stamp = timestamp()\n        >>> print('stamp = {!r}'.format(stamp))\n        stamp = ...-...-...T...\n    \"\"\"\n    if method == 'iso8601':\n        # ISO 8601\n        # datetime.datetime.utcnow().isoformat()\n        # datetime.datetime.now().isoformat()\n        # utcnow\n        tz_hour = time.timezone // 3600\n        utc_offset = str(tz_hour) if tz_hour < 0 else '+' + str(tz_hour)\n        stamp = time.strftime('%Y-%m-%dT%H%M%S') + utc_offset\n        return stamp\n    else:\n        raise ValueError('only iso8601 is accepted for now')"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef benchmark_hash_data():\n    import ubelt as ub\n    #ITEM = 'JUST A STRING' * 100\n    ITEM = [0, 1, 'a', 'b', ['JUST A STRING'] * 4]\n    HASHERS = ['sha1', 'sha512', 'xxh32', 'xxh64']\n    scales = list(range(5, 13))\n    results = ub.AutoDict()\n    # Use json is faster or at least as fast it most cases\n    # xxhash is also significantly faster than sha512\n    convert = ub.argval('--convert', default='True').lower() == 'True'\n    print('convert = {!r}'.format(convert))\n    ti = ub.Timerit(9, bestof=3, verbose=1, unit='ms')\n    for s in ub.ProgIter(scales, desc='benchmark', verbose=3):\n        N = 2 ** s\n        print(' --- s={s}, N={N} --- '.format(s=s, N=N))\n        data = [ITEM] * N\n        for hasher in HASHERS:\n            for timer in ti.reset(hasher):\n                ub.hash_data(data, hasher=hasher, convert=convert)\n            results[hasher].update({N: ti.mean()})\n        col = {h: results[h][N] for h in HASHERS}\n        sortx = ub.argsort(col)\n        ranking = ub.dict_subset(col, sortx)\n        print('walltime: ' + ub.repr2(ranking, precision=9, nl=0))\n        best = next(iter(ranking))\n        #pairs = list(ub.iter_window( 2))\n        pairs = [(k, best) for k in ranking]\n        ratios = [ranking[k1] / ranking[k2] for k1, k2 in pairs]\n        nicekeys = ['{}/{}'.format(k1, k2) for k1, k2 in pairs]\n        relratios = ub.odict(zip(nicekeys, ratios))\n        print('speedup: ' + ub.repr2(relratios, precision=4, nl=0))\n    # xdoc +REQUIRES(--show)\n    # import pytest\n    # pytest.skip()\n    import pandas as pd\n    df = pd.DataFrame.from_dict(results)\n    df.columns.name = 'hasher'\n    df.index.name = 'N'\n    ratios = df.copy().drop(columns=df.columns)\n    for k1, k2 in [('sha512', 'xxh32'), ('sha1', 'xxh32'), ('xxh64', 'xxh32')]:\n        ratios['{}/{}'.format(k1, k2)] = df[k1] / df[k2]\n    print()\n    print('Seconds per iteration')\n    print(df.to_string(float_format='%.9f'))\n    print()\n    print('Ratios of seconds')\n    print(ratios.to_string(float_format='%.2f'))\n    print()\n    print('Average Ratio (over all N)')\n    print('convert = {!r}'.format(convert))\n    print(ratios.mean().sort_values())\n    if ub.argflag('--show'):\n        import netharn.util as kwel\n        kwel.autompl()\n        xdata = sorted(ub.peek(results.values()).keys())\n        ydata = ub.map_vals(lambda d: [d[x] for x in xdata], results)\n        kwel.multi_plot(xdata, ydata, xlabel='N', ylabel='seconds', title='convert = {}'.format(convert))\n        kwel.show_if_requested()", "response": "Benchmark the data of the current n - tuple of items in a new n - tuple of n - tuples"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef import_module_from_path(modpath, index=-1):\n    import os\n    if not os.path.exists(modpath):\n        import re\n        import zipimport\n        # We allow (if not prefer or force) the colon to be a path.sep in order\n        # to agree with the mod.__name__ attribute that will be produced\n\n        # zip followed by colon or slash\n        pat = '(.zip[' + re.escape(os.path.sep) + '/:])'\n        parts = re.split(pat, modpath, flags=re.IGNORECASE)\n        if len(parts) > 2:\n            archivepath = ''.join(parts[:-1])[:-1]\n            internal = parts[-1]\n            modname = os.path.splitext(internal)[0]\n            modname = os.path.normpath(modname)\n            if os.path.exists(archivepath):\n                zimp_file = zipimport.zipimporter(archivepath)\n                module = zimp_file.load_module(modname)\n                return module\n        raise IOError('modpath={} does not exist'.format(modpath))\n    else:\n        # the importlib version doesnt work in pytest\n        module = _custom_import_modpath(modpath)\n        # TODO: use this implementation once pytest fixes importlib\n        # module = _pkgutil_import_modpath(modpath)\n        return module", "response": "Imports a module from a given path."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nimporting a module from its string name.", "response": "def import_module_from_name(modname):\n    \"\"\"\n    Imports a module from its string name (__name__)\n\n    Args:\n        modname (str):  module name\n\n    Returns:\n        module: module\n\n    Example:\n        >>> # test with modules that wont be imported in normal circumstances\n        >>> # todo write a test where we gaurentee this\n        >>> modname_list = [\n        >>>     'pickletools',\n        >>>     'lib2to3.fixes.fix_apply',\n        >>> ]\n        >>> #assert not any(m in sys.modules for m in modname_list)\n        >>> modules = [import_module_from_name(modname) for modname in modname_list]\n        >>> assert [m.__name__ for m in modules] == modname_list\n        >>> assert all(m in sys.modules for m in modname_list)\n    \"\"\"\n    if True:\n        # See if this fixes the Docker issue we saw but were unable to\n        # reproduce on another environment. Either way its better to use the\n        # standard importlib implementation than the one I wrote a long time\n        # ago.\n        import importlib\n        module = importlib.import_module(modname)\n    else:\n        # The __import__ statment is weird\n        if '.' in modname:\n            fromlist = modname.split('.')[-1]\n            fromlist_ = list(map(str, fromlist))  # needs to be ascii for python2.7\n            module = __import__(modname, {}, {}, fromlist_, 0)\n        else:\n            module = __import__(modname, {}, {}, [], 0)\n    return module"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _platform_pylib_exts():  # nocover\n    import sysconfig\n    valid_exts = []\n    if six.PY2:\n        # see also 'SHLIB_EXT'\n        base_ext = '.' + sysconfig.get_config_var('SO').split('.')[-1]\n    else:\n        # return with and without API flags\n        # handle PEP 3149 -- ABI version tagged .so files\n        base_ext = '.' + sysconfig.get_config_var('EXT_SUFFIX').split('.')[-1]\n    for tag in _extension_module_tags():\n        valid_exts.append('.' + tag + base_ext)\n    valid_exts.append(base_ext)\n    return tuple(valid_exts)", "response": "Return a tuple of valid extension names for the platform Python library."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning sys. path version of modname_to_modpath.", "response": "def _syspath_modname_to_modpath(modname, sys_path=None, exclude=None):\n    \"\"\"\n    syspath version of modname_to_modpath\n\n    Args:\n        modname (str): name of module to find\n        sys_path (List[PathLike], default=None):\n            if specified overrides `sys.path`\n        exclude (List[PathLike], default=None):\n            list of directory paths. if specified prevents these directories\n            from being searched.\n\n    Notes:\n        This is much slower than the pkgutil mechanisms.\n\n    CommandLine:\n        python -m xdoctest.static_analysis _syspath_modname_to_modpath\n\n    Example:\n        >>> print(_syspath_modname_to_modpath('xdoctest.static_analysis'))\n        ...static_analysis.py\n        >>> print(_syspath_modname_to_modpath('xdoctest'))\n        ...xdoctest\n        >>> print(_syspath_modname_to_modpath('_ctypes'))\n        ..._ctypes...\n        >>> assert _syspath_modname_to_modpath('xdoctest', sys_path=[]) is None\n        >>> assert _syspath_modname_to_modpath('xdoctest.static_analysis', sys_path=[]) is None\n        >>> assert _syspath_modname_to_modpath('_ctypes', sys_path=[]) is None\n        >>> assert _syspath_modname_to_modpath('this', sys_path=[]) is None\n\n    Example:\n        >>> # test what happens when the module is not visible in the path\n        >>> modname = 'xdoctest.static_analysis'\n        >>> modpath = _syspath_modname_to_modpath(modname)\n        >>> exclude = [split_modpath(modpath)[0]]\n        >>> found = _syspath_modname_to_modpath(modname, exclude=exclude)\n        >>> # this only works if installed in dev mode, pypi fails\n        >>> assert found is None, 'should not have found {}'.format(found)\n    \"\"\"\n\n    def _isvalid(modpath, base):\n        # every directory up to the module, should have an init\n        subdir = dirname(modpath)\n        while subdir and subdir != base:\n            if not exists(join(subdir, '__init__.py')):\n                return False\n            subdir = dirname(subdir)\n        return True\n\n    _fname_we = modname.replace('.', os.path.sep)\n    candidate_fnames = [\n        _fname_we + '.py',\n        # _fname_we + '.pyc',\n        # _fname_we + '.pyo',\n    ]\n    # Add extension library suffixes\n    candidate_fnames += [_fname_we + ext for ext in _platform_pylib_exts()]\n\n    if sys_path is None:\n        sys_path = sys.path\n\n    # the empty string in sys.path indicates cwd. Change this to a '.'\n    candidate_dpaths = ['.' if p == '' else p for p in sys_path]\n\n    if exclude:\n        def normalize(p):\n            if sys.platform.startswith('win32'):  # nocover\n                return realpath(p).lower()\n            else:\n                return realpath(p)\n        # Keep only the paths not in exclude\n        real_exclude = {normalize(p) for p in exclude}\n        candidate_dpaths = [p for p in candidate_dpaths\n                            if normalize(p) not in real_exclude]\n\n    for dpath in candidate_dpaths:\n        # Check for directory-based modules (has presidence over files)\n        modpath = join(dpath, _fname_we)\n        if exists(modpath):\n            if isfile(join(modpath, '__init__.py')):\n                if _isvalid(modpath, dpath):\n                    return modpath\n\n        # If that fails, check for file-based modules\n        for fname in candidate_fnames:\n            modpath = join(dpath, fname)\n            if isfile(modpath):\n                if _isvalid(modpath, dpath):\n                    return modpath"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the path to a python module from its name.", "response": "def modname_to_modpath(modname, hide_init=True, hide_main=False, sys_path=None):\n    \"\"\"\n    Finds the path to a python module from its name.\n\n    Determines the path to a python module without directly import it\n\n    Converts the name of a module (__name__) to the path (__file__) where it is\n    located without importing the module. Returns None if the module does not\n    exist.\n\n    Args:\n        modname (str): module filepath\n        hide_init (bool): if False, __init__.py will be returned for packages\n        hide_main (bool): if False, and hide_init is True, __main__.py will be\n            returned for packages, if it exists.\n        sys_path (list): if specified overrides `sys.path` (default None)\n\n    Returns:\n        str: modpath - path to the module, or None if it doesn't exist\n\n    CommandLine:\n        python -m xdoctest.static_analysis modname_to_modpath:0\n        pytest  /home/joncrall/code/xdoctest/xdoctest/static_analysis.py::modname_to_modpath:0\n\n    Example:\n        >>> modname = 'xdoctest.__main__'\n        >>> modpath = modname_to_modpath(modname, hide_main=False)\n        >>> assert modpath.endswith('__main__.py')\n        >>> modname = 'xdoctest'\n        >>> modpath = modname_to_modpath(modname, hide_init=False)\n        >>> assert modpath.endswith('__init__.py')\n        >>> modpath = basename(modname_to_modpath('_ctypes'))\n        >>> assert 'ctypes' in modpath\n    \"\"\"\n    modpath = _syspath_modname_to_modpath(modname, sys_path)\n    if modpath is None:\n        return None\n\n    modpath = normalize_modpath(modpath, hide_init=hide_init,\n                                hide_main=hide_main)\n    return modpath"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nnormalizes a module path.", "response": "def normalize_modpath(modpath, hide_init=True, hide_main=False):\n    \"\"\"\n    Normalizes __init__ and __main__ paths.\n\n    Notes:\n        Adds __init__ if reasonable, but only removes __main__ by default\n\n    Args:\n        hide_init (bool): if True, always return package modules\n           as __init__.py files otherwise always return the dpath.\n        hide_init (bool): if True, always strip away main files otherwise\n           ignore __main__.py.\n\n    CommandLine:\n        xdoctest -m xdoctest.static_analysis normalize_modpath\n\n    Example:\n        >>> import xdoctest.static_analysis as static\n        >>> modpath = static.__file__\n        >>> assert static.normalize_modpath(modpath) == modpath.replace('.pyc', '.py')\n        >>> dpath = dirname(modpath)\n        >>> res0 = static.normalize_modpath(dpath, hide_init=0, hide_main=0)\n        >>> res1 = static.normalize_modpath(dpath, hide_init=0, hide_main=1)\n        >>> res2 = static.normalize_modpath(dpath, hide_init=1, hide_main=0)\n        >>> res3 = static.normalize_modpath(dpath, hide_init=1, hide_main=1)\n        >>> assert res0.endswith('__init__.py')\n        >>> assert res1.endswith('__init__.py')\n        >>> assert not res2.endswith('.py')\n        >>> assert not res3.endswith('.py')\n    \"\"\"\n    if six.PY2:\n        if modpath.endswith('.pyc'):\n            modpath = modpath[:-1]\n    if hide_init:\n        if basename(modpath) == '__init__.py':\n            modpath = dirname(modpath)\n            hide_main = True\n    else:\n        # add in init, if reasonable\n        modpath_with_init = join(modpath, '__init__.py')\n        if exists(modpath_with_init):\n            modpath = modpath_with_init\n    if hide_main:\n        # We can remove main, but dont add it\n        if basename(modpath) == '__main__.py':\n            # corner case where main might just be a module name not in a pkg\n            parallel_init = join(dirname(modpath), '__init__.py')\n            if exists(parallel_init):\n                modpath = dirname(modpath)\n    return modpath"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef modpath_to_modname(modpath, hide_init=True, hide_main=False, check=True,\n                       relativeto=None):\n    \"\"\"\n    Determines importable name from file path\n\n    Converts the path to a module (__file__) to the importable python name\n    (__name__) without importing the module.\n\n    The filename is converted to a module name, and parent directories are\n    recursively included until a directory without an __init__.py file is\n    encountered.\n\n    Args:\n        modpath (str): module filepath\n        hide_init (bool): removes the __init__ suffix (default True)\n        hide_main (bool): removes the __main__ suffix (default False)\n        check (bool): if False, does not raise an error if modpath is a dir\n            and does not contain an __init__ file.\n        relativeto (str, optional): if specified, all checks are ignored and\n            this is considered the path to the root module.\n\n    Returns:\n        str: modname\n\n    Raises:\n        ValueError: if check is True and the path does not exist\n\n    CommandLine:\n        xdoctest -m xdoctest.static_analysis modpath_to_modname\n\n    Example:\n        >>> from xdoctest import static_analysis\n        >>> modpath = static_analysis.__file__.replace('.pyc', '.py')\n        >>> modpath = modpath.replace('.pyc', '.py')\n        >>> modname = modpath_to_modname(modpath)\n        >>> assert modname == 'xdoctest.static_analysis'\n\n    Example:\n        >>> import xdoctest\n        >>> assert modpath_to_modname(xdoctest.__file__.replace('.pyc', '.py')) == 'xdoctest'\n        >>> assert modpath_to_modname(dirname(xdoctest.__file__.replace('.pyc', '.py'))) == 'xdoctest'\n\n    Example:\n        >>> modpath = modname_to_modpath('_ctypes')\n        >>> modname = modpath_to_modname(modpath)\n        >>> assert modname == '_ctypes'\n    \"\"\"\n    if check and relativeto is None:\n        if not exists(modpath):\n            raise ValueError('modpath={} does not exist'.format(modpath))\n    modpath_ = abspath(expanduser(modpath))\n\n    modpath_ = normalize_modpath(modpath_, hide_init=hide_init,\n                                 hide_main=hide_main)\n    if relativeto:\n        dpath = dirname(abspath(expanduser(relativeto)))\n        rel_modpath = relpath(modpath_, dpath)\n    else:\n        dpath, rel_modpath = split_modpath(modpath_, check=check)\n\n    modname = splitext(rel_modpath)[0]\n    if '.' in modname:\n        modname, abi_tag = modname.split('.')\n    modname = modname.replace('/', '.')\n    modname = modname.replace('\\\\', '.')\n    return modname", "response": "Returns the importable name from file path"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef split_modpath(modpath, check=True):\n    if six.PY2:\n        if modpath.endswith('.pyc'):\n            modpath = modpath[:-1]\n    modpath_ = abspath(expanduser(modpath))\n    if check:\n        if not exists(modpath_):\n            if not exists(modpath):\n                raise ValueError('modpath={} does not exist'.format(modpath))\n            raise ValueError('modpath={} is not a module'.format(modpath))\n        if isdir(modpath_) and not exists(join(modpath, '__init__.py')):\n            # dirs without inits are not modules\n            raise ValueError('modpath={} is not a module'.format(modpath))\n    full_dpath, fname_ext = split(modpath_)\n    _relmod_parts = [fname_ext]\n    # Recurse down directories until we are out of the package\n    dpath = full_dpath\n    while exists(join(dpath, '__init__.py')):\n        dpath, dname = split(dpath)\n        _relmod_parts.append(dname)\n    relmod_parts = _relmod_parts[::-1]\n    rel_modpath = os.path.sep.join(relmod_parts)\n    return dpath, rel_modpath", "response": "Splits the module path into a tuple of directories and relative paths."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ngets the value of a keyword argument specified on the command line.", "response": "def argval(key, default=util_const.NoParam, argv=None):\n    \"\"\"\n    Get the value of a keyword argument specified on the command line.\n\n    Values can be specified as `<key> <value>` or `<key>=<value>`\n\n    Args:\n        key (str or tuple): string or tuple of strings. Each key should be\n            prefixed with two hyphens (i.e. `--`)\n        default (Optional[object]): value to return if not specified\n        argv (Optional[list]): overrides `sys.argv` if specified\n\n    Returns:\n        str: value : the value specified after the key. It they key is\n            specified multiple times, then the first value is returned.\n\n    TODO:\n        - [ ] Can we handle the case where the value is a list of long paths?\n        - [ ] Should we default the first or last specified instance of the flag.\n\n    Example:\n        >>> import ubelt as ub\n        >>> argv = ['--ans', '42', '--quest=the grail', '--ans=6', '--bad']\n        >>> assert ub.argval('--spam', argv=argv) == ub.NoParam\n        >>> assert ub.argval('--quest', argv=argv) == 'the grail'\n        >>> assert ub.argval('--ans', argv=argv) == '42'\n        >>> assert ub.argval('--bad', argv=argv) == ub.NoParam\n        >>> assert ub.argval(('--bad', '--bar'), argv=argv) == ub.NoParam\n\n    Example:\n        >>> # Test fix for GH Issue #41\n        >>> import ubelt as ub\n        >>> argv = ['--path=/path/with/k=3']\n        >>> ub.argval('--path', argv=argv) == '/path/with/k=3'\n    \"\"\"\n    if argv is None:  # nocover\n        argv = sys.argv\n\n    keys = [key] if isinstance(key, six.string_types) else key\n    n_max = len(argv) - 1\n    for argx, item in enumerate(argv):\n        for key_ in keys:\n            if item == key_:\n                if argx < n_max:\n                    value = argv[argx + 1]\n                    return value\n            elif item.startswith(key_ + '='):\n                value = '='.join(item.split('=')[1:])\n                return value\n    value = default\n    return value"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef argflag(key, argv=None):\n    if argv is None:  # nocover\n        argv = sys.argv\n    keys = [key] if isinstance(key, six.string_types) else key\n    flag = any(k in argv for k in keys)\n    return flag", "response": "Determines if a key is specified on the command line."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef hzcat(args, sep=''):\n    import unicodedata\n    if '\\n' in sep or '\\r' in sep:\n        raise ValueError('`sep` cannot contain newline characters')\n\n    # TODO: ensure unicode data works correctly for python2\n    args = [unicodedata.normalize('NFC', ensure_unicode(val)) for val in args]\n    arglines = [a.split('\\n') for a in args]\n    height = max(map(len, arglines))\n    # Do vertical padding\n    arglines = [lines + [''] * (height - len(lines)) for lines in arglines]\n    # Initialize output\n    all_lines = ['' for _ in range(height)]\n    width = 0\n    n_args = len(args)\n    for sx, lines in enumerate(arglines):\n        # Concatenate the new string\n        for lx, line in enumerate(lines):\n            all_lines[lx] += line\n        # Find the new maximum horizontal width\n        width = max(width, max(map(len, all_lines)))\n        if sx < n_args - 1:\n            # Horizontal padding on all but last iter\n            for lx, line in list(enumerate(all_lines)):\n                residual = width - len(line)\n                all_lines[lx] = line + (' ' * residual) + sep\n            width += len(sep)\n    # Clean up trailing whitespace\n    all_lines = [line.rstrip(' ') for line in all_lines]\n    ret = '\\n'.join(all_lines)\n    return ret", "response": "Horizontally concatenates strings preserving indentation and indentation"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef symlink(real_path, link_path, overwrite=False, verbose=0):\n    path = normpath(real_path)\n    link = normpath(link_path)\n\n    if not os.path.isabs(path):\n        # if path is not absolute it must be specified relative to link\n        if _can_symlink():\n            path = os.path.relpath(path, os.path.dirname(link))\n        else:  # nocover\n            # On windows, we need to use absolute paths\n            path = os.path.abspath(path)\n\n    if verbose:\n        print('Symlink: {path} -> {link}'.format(path=path, link=link))\n    if islink(link):\n        if verbose:\n            print('... already exists')\n        pointed = _readlink(link)\n        if pointed == path:\n            if verbose > 1:\n                print('... and points to the right place')\n            return link\n        if verbose > 1:\n            if not exists(link):\n                print('... but it is broken and points somewhere else: {}'.format(pointed))\n            else:\n                print('... but it points somewhere else: {}'.format(pointed))\n        if overwrite:\n            util_io.delete(link, verbose=verbose > 1)\n    elif exists(link):\n        if _win32_links is None:\n            if verbose:\n                print('... already exists, but its a file. This will error.')\n            raise FileExistsError(\n                'cannot overwrite a physical path: \"{}\"'.format(path))\n        else:  # nocover\n            if verbose:\n                print('... already exists, and is either a file or hard link. '\n                      'Assuming it is a hard link. '\n                      'On non-win32 systems this would error.')\n\n    if _win32_links is None:\n        os.symlink(path, link)\n    else:  # nocover\n        _win32_links._symlink(path, link, overwrite=overwrite, verbose=verbose)\n\n    return link", "response": "Create a symbolic link."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ntest helper for printing directory information (mostly for investigating windows weirdness) CommandLine: python -m ubelt.util_links _dirstats", "response": "def _dirstats(dpath=None):  # nocover\n    \"\"\"\n    Testing helper for printing directory information\n    (mostly for investigating windows weirdness)\n\n    CommandLine:\n        python -m ubelt.util_links _dirstats\n    \"\"\"\n    from ubelt import util_colors\n    if dpath is None:\n        dpath = os.getcwd()\n    print('===============')\n    print('Listing for dpath={}'.format(dpath))\n    print('E L F D J - path')\n    print('--------------')\n    if not os.path.exists(dpath):\n        print('... does not exist')\n        return\n    paths = sorted(os.listdir(dpath))\n    for path in paths:\n        full_path = join(dpath, path)\n        E = os.path.exists(full_path)\n        L = os.path.islink(full_path)\n        F = os.path.isfile(full_path)\n        D = os.path.isdir(full_path)\n        J = util_platform.WIN32 and _win32_links._win32_is_junction(full_path)\n        ELFDJ = [E, L, F, D, J]\n        if   ELFDJ == [1, 0, 0, 1, 0]:\n            # A directory\n            path = util_colors.color_text(path, 'green')\n        elif ELFDJ == [1, 0, 1, 0, 0]:\n            # A file (or a hard link they are indistinguishable with one query)\n            path = util_colors.color_text(path, 'white')\n        elif ELFDJ == [1, 0, 0, 1, 1]:\n            # A directory junction\n            path = util_colors.color_text(path, 'yellow')\n        elif ELFDJ == [1, 1, 1, 0, 0]:\n            # A file link\n            path = util_colors.color_text(path, 'turquoise')\n        elif ELFDJ == [1, 1, 0, 1, 0]:\n            # A directory link\n            path = util_colors.color_text(path, 'teal')\n        elif ELFDJ == [0, 1, 0, 0, 0]:\n            # A broken file link\n            path = util_colors.color_text(path, 'red')\n        elif ELFDJ == [0, 1, 0, 1, 0]:\n            # A broken directory link\n            path = util_colors.color_text(path, 'darkred')\n        elif ELFDJ == [0, 0, 0, 1, 1]:\n            # A broken directory junction\n            path = util_colors.color_text(path, 'purple')\n        elif ELFDJ == [1, 0, 1, 0, 1]:\n            # A file junction? Thats not good.\n            # I guess this is a windows 7 thing?\n            path = util_colors.color_text(path, 'red')\n        elif ELFDJ == [1, 1, 0, 0, 0]:\n            # Windows? Why? What does this mean!?\n            # A directory link that cant be resolved?\n            path = util_colors.color_text(path, 'red')\n        else:\n            print('dpath = {!r}'.format(dpath))\n            print('path = {!r}'.format(path))\n            raise AssertionError(str(ELFDJ) + str(path))\n        line = '{E:d} {L:d} {F:d} {D:d} {J:d} - {path}'.format(**locals())\n        if os.path.islink(full_path):\n            line += ' -> ' + os.readlink(full_path)\n        elif _win32_links is not None:\n            if _win32_links._win32_is_junction(full_path):\n                line += ' => ' + _win32_links._win32_read_junction(full_path)\n        print(line)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _make_signature_key(args, kwargs):\n    kwitems = kwargs.items()\n    # TODO: we should check if Python is at least 3.7 and sort by kwargs\n    # keys otherwise. Should we use hash_data for key generation\n    if (sys.version_info.major, sys.version_info.minor) < (3, 7):  # nocover\n        # We can sort because they keys are gaurenteed to be strings\n        kwitems = sorted(kwitems)\n    kwitems = tuple(kwitems)\n\n    try:\n        key = _hashable(args), _hashable(kwitems)\n    except TypeError:\n        raise TypeError('Signature is not hashable: args={} kwargs{}'.format(args, kwargs))\n    return key", "response": "Returns a key that can be used by the cache\n       ."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a property attribute for new-style classes that only calls its getter on the first access. The result is stored and on subsequent accesses is returned, preventing the need to call the getter any more. This decorator can either be used by itself or by decorating another property. In either case the method will always become a property. Notes: implementation is a modified version of [1]. References: ..[1] https://github.com/estebistec/python-memoized-property CommandLine: xdoctest -m ubelt.util_memoize memoize_property Example: >>> class C(object): ... load_name_count = 0 ... @memoize_property ... def name(self): ... \"name's docstring\" ... self.load_name_count += 1 ... return \"the name\" ... @memoize_property ... @property ... def another_name(self): ... \"name's docstring\" ... self.load_name_count += 1 ... return \"the name\" >>> c = C() >>> c.load_name_count 0 >>> c.name 'the name' >>> c.load_name_count 1 >>> c.name 'the name' >>> c.load_name_count 1 >>> c.another_name", "response": "def memoize_property(fget):\n    \"\"\"\n    Return a property attribute for new-style classes that only calls its\n    getter on the first access. The result is stored and on subsequent accesses\n    is returned, preventing the need to call the getter any more.\n\n    This decorator can either be used by itself or by decorating another\n    property. In either case the method will always become a property.\n\n    Notes:\n        implementation is a modified version of [1].\n\n    References:\n        ..[1] https://github.com/estebistec/python-memoized-property\n\n    CommandLine:\n        xdoctest -m ubelt.util_memoize memoize_property\n\n    Example:\n        >>> class C(object):\n        ...     load_name_count = 0\n        ...     @memoize_property\n        ...     def name(self):\n        ...         \"name's docstring\"\n        ...         self.load_name_count += 1\n        ...         return \"the name\"\n        ...     @memoize_property\n        ...     @property\n        ...     def another_name(self):\n        ...         \"name's docstring\"\n        ...         self.load_name_count += 1\n        ...         return \"the name\"\n        >>> c = C()\n        >>> c.load_name_count\n        0\n        >>> c.name\n        'the name'\n        >>> c.load_name_count\n        1\n        >>> c.name\n        'the name'\n        >>> c.load_name_count\n        1\n        >>> c.another_name\n    \"\"\"\n    # Unwrap any existing property decorator\n    while hasattr(fget, 'fget'):\n        fget = fget.fget\n\n    attr_name = '_' + fget.__name__\n\n    @functools.wraps(fget)\n    def fget_memoized(self):\n        if not hasattr(self, attr_name):\n            setattr(self, attr_name, fget(self))\n        return getattr(self, attr_name)\n\n    return property(fget_memoized)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nhighlights a block of text using ANSI tags based on language syntax.", "response": "def highlight_code(text, lexer_name='python', **kwargs):\n    \"\"\"\n    Highlights a block of text using ANSI tags based on language syntax.\n\n    Args:\n        text (str): plain text to highlight\n        lexer_name (str): name of language\n        **kwargs: passed to pygments.lexers.get_lexer_by_name\n\n    Returns:\n        str: text : highlighted text\n            If pygments is not installed, the plain text is returned.\n\n    CommandLine:\n        python -c \"import pygments.formatters; print(list(pygments.formatters.get_all_formatters()))\"\n\n    Example:\n        >>> import ubelt as ub\n        >>> text = 'import ubelt as ub; print(ub)'\n        >>> new_text = ub.highlight_code(text)\n        >>> print(new_text)\n    \"\"\"\n    # Resolve extensions to languages\n    lexer_name = {\n        'py': 'python',\n        'h': 'cpp',\n        'cpp': 'cpp',\n        'cxx': 'cpp',\n        'c': 'cpp',\n    }.get(lexer_name.replace('.', ''), lexer_name)\n    try:\n        import pygments\n        import pygments.lexers\n        import pygments.formatters\n        import pygments.formatters.terminal\n\n        if sys.platform.startswith('win32'):  # nocover\n            # Hack on win32 to support colored output\n            import colorama\n            colorama.init()\n\n        formater = pygments.formatters.terminal.TerminalFormatter(bg='dark')\n        lexer = pygments.lexers.get_lexer_by_name(lexer_name, **kwargs)\n        new_text = pygments.highlight(text, lexer, formater)\n\n    except ImportError:  # nocover\n        import warnings\n        warnings.warn('pygments is not installed, code will not be highlighted')\n        new_text = text\n    return new_text"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking if the input is an iterable.", "response": "def iterable(obj, strok=False):\n    \"\"\"\n    Checks if the input implements the iterator interface. An exception is made\n    for strings, which return False unless `strok` is True\n\n    Args:\n        obj (object): a scalar or iterable input\n\n        strok (bool): if True allow strings to be interpreted as iterable\n\n    Returns:\n        bool: True if the input is iterable\n\n    Example:\n        >>> obj_list = [3, [3], '3', (3,), [3, 4, 5], {}]\n        >>> result = [iterable(obj) for obj in obj_list]\n        >>> assert result == [False, True, False, True, True, True]\n        >>> result = [iterable(obj, strok=True) for obj in obj_list]\n        >>> assert result == [False, True, True, True, True, True]\n    \"\"\"\n    try:\n        iter(obj)\n    except Exception:\n        return False\n    else:\n        return strok or not isinstance(obj, six.string_types)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef unique(items, key=None):\n    seen = set()\n    if key is None:\n        for item in items:\n            if item not in seen:\n                seen.add(item)\n                yield item\n    else:\n        for item in items:\n            norm = key(item)\n            if norm not in seen:\n                seen.add(norm)\n                yield item", "response": "Returns a list of unique items in the order they appear in the input sequence."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns indices corresponding to the first instance of each unique item in items.", "response": "def argunique(items, key=None):\n    \"\"\"\n    Returns indices corresponding to the first instance of each unique item.\n\n    Args:\n        items (Sequence): indexable collection of items\n\n        key (Callable, optional): custom normalization function.\n            If specified returns items where `key(item)` is unique.\n\n    Yields:\n        int : indices of the unique items\n\n    Example:\n        >>> items = [0, 2, 5, 1, 1, 0, 2, 4]\n        >>> indices = list(argunique(items))\n        >>> assert indices == [0, 1, 2, 3, 7]\n        >>> indices = list(argunique(items, key=lambda x: x % 2 == 0))\n        >>> assert indices == [0, 2]\n    \"\"\"\n    # yield from unique(range(len(items)), key=lambda i: items[i])\n    if key is None:\n        return unique(range(len(items)), key=lambda i: items[i])\n    else:\n        return unique(range(len(items)), key=lambda i: key(items[i]))"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a list of booleans corresponding to the first instance of each unique item.", "response": "def unique_flags(items, key=None):\n    \"\"\"\n    Returns a list of booleans corresponding to the first instance of each\n    unique item.\n\n    Args:\n        items (Sequence): indexable collection of items\n\n        key (Callable, optional): custom normalization function.\n            If specified returns items where `key(item)` is unique.\n\n    Returns:\n        List[bool] : flags the items that are unique\n\n    Example:\n        >>> import ubelt as ub\n        >>> items = [0, 2, 1, 1, 0, 9, 2]\n        >>> flags = unique_flags(items)\n        >>> assert flags == [True, True, True, False, False, True, False]\n        >>> flags = unique_flags(items, key=lambda x: x % 2 == 0)\n        >>> assert flags == [True, False, True, False, False, False, False]\n    \"\"\"\n    len_ = len(items)\n    if key is None:\n        item_to_index = dict(zip(reversed(items), reversed(range(len_))))\n        indices = item_to_index.values()\n    else:\n        indices = argunique(items, key=key)\n    flags = boolmask(indices, len_)\n    return flags"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef boolmask(indices, maxval=None):\n    if maxval is None:\n        indices = list(indices)\n        maxval = max(indices) + 1\n    mask = [False] * maxval\n    for index in indices:\n        mask[index] = True\n    return mask", "response": "Construct a list of booleans where an item is True if its position is in\n    otherwise it is False."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn True if all items in a sequence are the same as the same one.", "response": "def allsame(iterable, eq=operator.eq):\n    \"\"\"\n    Determine if all items in a sequence are the same\n\n    Args:\n        iterable (Iterable): items to determine if they are all the same\n\n        eq (Callable, optional): function to determine equality\n            (default: operator.eq)\n\n    Example:\n        >>> allsame([1, 1, 1, 1])\n        True\n        >>> allsame([])\n        True\n        >>> allsame([0, 1])\n        False\n        >>> iterable = iter([0, 1, 1, 1])\n        >>> next(iterable)\n        >>> allsame(iterable)\n        True\n        >>> allsame(range(10))\n        False\n        >>> allsame(range(10), lambda a, b: True)\n        True\n    \"\"\"\n    iter_ = iter(iterable)\n    try:\n        first = next(iter_)\n    except StopIteration:\n        return True\n    return all(eq(first, item) for item in iter_)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef argsort(indexable, key=None, reverse=False):\n    # Create an iterator of value/key pairs\n    if isinstance(indexable, collections_abc.Mapping):\n        vk_iter = ((v, k) for k, v in indexable.items())\n    else:\n        vk_iter = ((v, k) for k, v in enumerate(indexable))\n    # Sort by values and extract the indices\n    if key is None:\n        indices = [k for v, k in sorted(vk_iter, reverse=reverse)]\n    else:\n        # If key is provided, call it using the value as input\n        indices = [k for v, k in sorted(vk_iter, key=lambda vk: key(vk[0]),\n                                        reverse=reverse)]\n    return indices", "response": "Returns the indices that would sort a indexable object."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning index or key of the item with the largest value.", "response": "def argmax(indexable, key=None):\n    \"\"\"\n    Returns index / key of the item with the largest value.\n\n    This is similar to `numpy.argmax`, but it is written in pure python and\n    works on both lists and dictionaries.\n\n    Args:\n        indexable (Iterable or Mapping): indexable to sort by\n\n        key (Callable, optional): customizes the ordering of the indexable\n\n    CommandLine:\n        python -m ubelt.util_list argmax\n\n    Example:\n        >>> assert argmax({'a': 3, 'b': 2, 'c': 100}) == 'c'\n        >>> assert argmax(['a', 'c', 'b', 'z', 'f']) == 3\n        >>> assert argmax([[0, 1], [2, 3, 4], [5]], key=len) == 1\n        >>> assert argmax({'a': 3, 'b': 2, 3: 100, 4: 4}) == 3\n        >>> assert argmax(iter(['a', 'c', 'b', 'z', 'f'])) == 3\n    \"\"\"\n    if key is None and isinstance(indexable, collections_abc.Mapping):\n        return max(indexable.items(), key=operator.itemgetter(1))[0]\n    elif hasattr(indexable, 'index'):\n        if key is None:\n            return indexable.index(max(indexable))\n        else:\n            return indexable.index(max(indexable, key=key))\n    else:\n        # less efficient, but catch all solution\n        return argsort(indexable, key=key)[-1]"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn index / key of the item with the smallest value.", "response": "def argmin(indexable, key=None):\n    \"\"\"\n    Returns index / key of the item with the smallest value.\n\n    This is similar to `numpy.argmin`, but it is written in pure python and\n    works on both lists and dictionaries.\n\n    Args:\n        indexable (Iterable or Mapping): indexable to sort by\n\n        key (Callable, optional): customizes the ordering of the indexable\n\n    Example:\n        >>> assert argmin({'a': 3, 'b': 2, 'c': 100}) == 'b'\n        >>> assert argmin(['a', 'c', 'b', 'z', 'f']) == 0\n        >>> assert argmin([[0, 1], [2, 3, 4], [5]], key=len) == 2\n        >>> assert argmin({'a': 3, 'b': 2, 3: 100, 4: 4}) == 'b'\n        >>> assert argmin(iter(['a', 'c', 'A', 'z', 'f'])) == 2\n    \"\"\"\n    if key is None and isinstance(indexable, collections_abc.Mapping):\n        return min(indexable.items(), key=operator.itemgetter(1))[0]\n    elif hasattr(indexable, 'index'):\n        if key is None:\n            return indexable.index(min(indexable))\n        else:\n            return indexable.index(min(indexable, key=key))\n    else:\n        # less efficient, but catch all solution\n        return argsort(indexable, key=key)[0]"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn a dictionary that associates the elements of items1 and items2 into items1.", "response": "def dzip(items1, items2, cls=dict):\n    \"\"\"\n    Zips elementwise pairs between items1 and items2 into a dictionary. Values\n    from items2 can be broadcast onto items1.\n\n    Args:\n        items1 (Iterable): full sequence\n        items2 (Iterable): can either be a sequence of one item or a sequence\n            of equal length to `items1`\n        cls (Type[dict]): dictionary type to use. Defaults to dict, but could\n            be ordered dict instead.\n\n    Returns:\n        dict: similar to dict(zip(items1, items2))\n\n    Example:\n        >>> assert dzip([1, 2, 3], [4]) == {1: 4, 2: 4, 3: 4}\n        >>> assert dzip([1, 2, 3], [4, 4, 4]) == {1: 4, 2: 4, 3: 4}\n        >>> assert dzip([], [4]) == {}\n    \"\"\"\n    try:\n        len(items1)\n    except TypeError:\n        items1 = list(items1)\n    try:\n        len(items2)\n    except TypeError:\n        items2 = list(items2)\n    if len(items1) == 0 and len(items2) == 1:\n        # Corner case:\n        # allow the first list to be empty and the second list to broadcast a\n        # value. This means that the equality check wont work for the case\n        # where items1 and items2 are supposed to correspond, but the length of\n        # items2 is 1.\n        items2 = []\n    if len(items2) == 1 and len(items1) > 1:\n        items2 = items2 * len(items1)\n    if len(items1) != len(items2):\n        raise ValueError('out of alignment len(items1)=%r, len(items2)=%r' % (\n            len(items1), len(items2)))\n    return cls(zip(items1, items2))"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef group_items(items, groupids):\n    if callable(groupids):\n        keyfunc = groupids\n        pair_list = ((keyfunc(item), item) for item in items)\n    else:\n        pair_list = zip(groupids, items)\n\n    # Initialize a dict of lists\n    groupid_to_items = defaultdict(list)\n    # Insert each item into the correct group\n    for key, item in pair_list:\n        groupid_to_items[key].append(item)\n    return groupid_to_items", "response": "r Returns a dict mapping a groupid to a list of items."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nbuilding a histogram of items, counting the number of time each item appears in the input. Args: item_list (Iterable): hashable items (usually containing duplicates) weight_list (Iterable): corresponding weights for each item ordered (bool): if True the result is ordered by frequency labels (Iterable, optional): expected labels (default None) Allows this function to pre-initialize the histogram. If specified the frequency of each label is initialized to zero and item_list can only contain items specified in labels. Returns: dict : dictionary where the keys are items in item_list, and the values are the number of times the item appears in item_list. CommandLine: python -m ubelt.util_dict dict_hist Example: >>> import ubelt as ub >>> item_list = [1, 2, 39, 900, 1232, 900, 1232, 2, 2, 2, 900] >>> hist = ub.dict_hist(item_list) >>> print(ub.repr2(hist, nl=0)) {1: 1, 2: 4, 39: 1, 900: 3, 1232: 2} Example: >>> import ubelt as ub >>> item_list = [1, 2, 39, 900, 1232, 900, 1232, 2, 2, 2, 900] >>> hist1 = ub.dict_hist(item_list) >>> hist2 = ub.dict_hist(item_list, ordered=True) >>> try: >>> hist3 = ub.dict_hist(item_list, labels=[]) >>> except KeyError: >>> pass >>> else: >>> raise AssertionError('expected key error') >>> #result = ub.repr2(hist_) >>> weight_list = [1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1] >>> hist4 = ub.dict_hist(item_list, weight_list=weight_list) >>> print(ub.repr2(hist1, nl=0)) {1: 1, 2: 4, 39: 1, 900: 3, 1232: 2} >>> print(ub.repr2(hist4, nl=0)) {1: 1, 2: 4, 39: 1, 900: 1, 1232: 0}", "response": "def dict_hist(item_list, weight_list=None, ordered=False, labels=None):\n    \"\"\"\n    Builds a histogram of items, counting the number of time each item appears\n    in the input.\n\n    Args:\n        item_list (Iterable): hashable items (usually containing duplicates)\n        weight_list (Iterable): corresponding weights for each item\n        ordered (bool): if True the result is ordered by frequency\n        labels (Iterable, optional): expected labels (default None)\n            Allows this function to pre-initialize the histogram.\n            If specified the frequency of each label is initialized to\n            zero and item_list can only contain items specified in labels.\n\n    Returns:\n        dict : dictionary where the keys are items in item_list, and the values\n          are the number of times the item appears in item_list.\n\n    CommandLine:\n        python -m ubelt.util_dict dict_hist\n\n    Example:\n        >>> import ubelt as ub\n        >>> item_list = [1, 2, 39, 900, 1232, 900, 1232, 2, 2, 2, 900]\n        >>> hist = ub.dict_hist(item_list)\n        >>> print(ub.repr2(hist, nl=0))\n        {1: 1, 2: 4, 39: 1, 900: 3, 1232: 2}\n\n    Example:\n        >>> import ubelt as ub\n        >>> item_list = [1, 2, 39, 900, 1232, 900, 1232, 2, 2, 2, 900]\n        >>> hist1 = ub.dict_hist(item_list)\n        >>> hist2 = ub.dict_hist(item_list, ordered=True)\n        >>> try:\n        >>>     hist3 = ub.dict_hist(item_list, labels=[])\n        >>> except KeyError:\n        >>>     pass\n        >>> else:\n        >>>     raise AssertionError('expected key error')\n        >>> #result = ub.repr2(hist_)\n        >>> weight_list = [1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1]\n        >>> hist4 = ub.dict_hist(item_list, weight_list=weight_list)\n        >>> print(ub.repr2(hist1, nl=0))\n        {1: 1, 2: 4, 39: 1, 900: 3, 1232: 2}\n        >>> print(ub.repr2(hist4, nl=0))\n        {1: 1, 2: 4, 39: 1, 900: 1, 1232: 0}\n    \"\"\"\n    if labels is None:\n        hist_ = defaultdict(lambda: 0)\n    else:\n        hist_ = {k: 0 for k in labels}\n    if weight_list is None:\n        weight_list = it.repeat(1)\n    # Accumulate frequency\n    for item, weight in zip(item_list, weight_list):\n        hist_[item] += weight\n    if ordered:\n        # Order by value\n        getval = op.itemgetter(1)\n        hist = OrderedDict([\n            (key, value)\n            for (key, value) in sorted(hist_.items(), key=getval)\n        ])\n    else:\n        # Cast to a normal dictionary\n        hist = dict(hist_)\n    return hist"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef find_duplicates(items, k=2, key=None):\n    # Build mapping from items to the indices at which they appear\n    # if key is not None:\n    #     items = map(key, items)\n    duplicates = defaultdict(list)\n    if key is None:\n        for count, item in enumerate(items):\n            duplicates[item].append(count)\n    else:\n        for count, item in enumerate(items):\n            duplicates[key(item)].append(count)\n    # remove items seen fewer than k times.\n    for key in list(duplicates.keys()):\n        if len(duplicates[key]) < k:\n            del duplicates[key]\n    duplicates = dict(duplicates)\n    return duplicates", "response": "Find all duplicate items in a list and return a mapping\n        from each duplicate item to the positions at which it appears in."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef dict_take(dict_, keys, default=util_const.NoParam):\n    if default is util_const.NoParam:\n        for key in keys:\n            yield dict_[key]\n    else:\n        for key in keys:\n            yield dict_.get(key, default)", "response": "r Returns a generator that yields the values from a dictionary."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns a dict that combines the disjoint keys in multiple dictionaries.", "response": "def dict_union(*args):\n    \"\"\"\n    Combines the disjoint keys in multiple dictionaries. For intersecting keys,\n    dictionaries towards the end of the sequence are given precedence.\n\n    Args:\n        *args : a sequence of dictionaries\n\n    Returns:\n        Dict | OrderedDict :\n            OrderedDict if the first argument is an OrderedDict, otherwise dict\n\n    SeeAlso:\n        collections.ChainMap - a standard python builtin data structure that\n            provides a view that treats multiple dicts as a single dict.\n            https://docs.python.org/3/library/collections.html#chainmap-objects\n\n    Example:\n        >>> result = dict_union({'a': 1, 'b': 1}, {'b': 2, 'c': 2})\n        >>> assert result == {'a': 1, 'b': 2, 'c': 2}\n        >>> dict_union(odict([('a', 1), ('b', 2)]), odict([('c', 3), ('d', 4)]))\n        OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 4)])\n        >>> dict_union()\n        {}\n    \"\"\"\n    if not args:\n        return {}\n    else:\n        dictclass = OrderedDict if isinstance(args[0], OrderedDict) else dict\n        return dictclass(it.chain.from_iterable(d.items() for d in args))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a dictionary that contains keys common between all inputs.", "response": "def dict_isect(*args):\n    \"\"\"\n    Constructs a dictionary that contains keys common between all inputs.\n    The returned values will only belong to the first dictionary.\n\n    Args:\n        *args : a sequence of dictionaries (or sets of keys)\n\n    Returns:\n        Dict | OrderedDict :\n            OrderedDict if the first argument is an OrderedDict, otherwise dict\n\n    Notes:\n        This function can be used as an alternative to `dict_subset` where any\n        key not in the dictionary is ignored. See the following example:\n\n        >>> dict_isect({'a': 1, 'b': 2, 'c': 3}, ['a', 'c', 'd'])\n        {'a': 1, 'c': 3}\n\n    Example:\n        >>> dict_isect({'a': 1, 'b': 1}, {'b': 2, 'c': 2})\n        {'b': 1}\n        >>> dict_isect(odict([('a', 1), ('b', 2)]), odict([('c', 3)]))\n        OrderedDict()\n        >>> dict_isect()\n        {}\n    \"\"\"\n    if not args:\n        return {}\n    else:\n        dictclass = OrderedDict if isinstance(args[0], OrderedDict) else dict\n        common_keys = set.intersection(*map(set, args))\n        first_dict = args[0]\n        return dictclass((k, first_dict[k]) for k in common_keys)"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\napplies a function to each of the keys in a dictionary and returns a new dictionary with the transformed keys.", "response": "def map_vals(func, dict_):\n    \"\"\"\n    applies a function to each of the keys in a dictionary\n\n    Args:\n        func (callable): a function or indexable object\n        dict_ (dict): a dictionary\n\n    Returns:\n        newdict: transformed dictionary\n\n    CommandLine:\n        python -m ubelt.util_dict map_vals\n\n    Example:\n        >>> import ubelt as ub\n        >>> dict_ = {'a': [1, 2, 3], 'b': []}\n        >>> func = len\n        >>> newdict = ub.map_vals(func, dict_)\n        >>> assert newdict ==  {'a': 3, 'b': 0}\n        >>> print(newdict)\n        >>> # Can also use indexables as `func`\n        >>> dict_ = {'a': 0, 'b': 1}\n        >>> func = [42, 21]\n        >>> newdict = ub.map_vals(func, dict_)\n        >>> assert newdict ==  {'a': 42, 'b': 21}\n        >>> print(newdict)\n    \"\"\"\n    if not hasattr(func, '__call__'):\n        func = func.__getitem__\n    keyval_list = [(key, func(val)) for key, val in six.iteritems(dict_)]\n    dictclass = OrderedDict if isinstance(dict_, OrderedDict) else dict\n    newdict = dictclass(keyval_list)\n    # newdict = type(dict_)(keyval_list)\n    return newdict"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef to_dict(self):\n        return self._base(\n            (key, (value.to_dict() if isinstance(value, AutoDict) else value))\n            for key, value in self.items())", "response": "Recursively casts a AutoDict into a regular dictionary. All nested\n            values are also converted."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _win32_can_symlink(verbose=0, force=0, testing=0):\n    global __win32_can_symlink__\n    if verbose:\n        print('__win32_can_symlink__ = {!r}'.format(__win32_can_symlink__))\n    if __win32_can_symlink__ is not None and not force:\n        return __win32_can_symlink__\n\n    from ubelt import util_platform\n    tempdir = util_platform.ensure_app_cache_dir('ubelt', '_win32_can_symlink')\n\n    util_io.delete(tempdir)\n    util_path.ensuredir(tempdir)\n\n    dpath = join(tempdir, 'dpath')\n    fpath = join(tempdir, 'fpath.txt')\n\n    dlink = join(tempdir, 'dlink')\n    flink = join(tempdir, 'flink.txt')\n\n    util_path.ensuredir(dpath)\n    util_io.touch(fpath)\n\n    # Add broken variants of the links for testing purposes\n    # Its ugly, but so is all this windows code.\n    if testing:\n        broken_dpath = join(tempdir, 'broken_dpath')\n        broken_fpath = join(tempdir, 'broken_fpath.txt')\n        # Create files that we will delete after we link to them\n        util_path.ensuredir(broken_dpath)\n        util_io.touch(broken_fpath)\n\n    try:\n        _win32_symlink(dpath, dlink)\n        if testing:\n            _win32_symlink(broken_dpath, join(tempdir, 'broken_dlink'))\n        can_symlink_directories = os.path.islink(dlink)\n    except OSError:\n        can_symlink_directories = False\n    if verbose:\n        print('can_symlink_directories = {!r}'.format(can_symlink_directories))\n\n    try:\n        _win32_symlink(fpath, flink)\n        if testing:\n            _win32_symlink(broken_fpath, join(tempdir, 'broken_flink'))\n        can_symlink_files = os.path.islink(flink)\n        # os.path.islink(flink)\n    except OSError:\n        can_symlink_files = False\n    if verbose:\n        print('can_symlink_files = {!r}'.format(can_symlink_files))\n\n    assert int(can_symlink_directories) + int(can_symlink_files) != 1, (\n        'can do one but not both. Unexpected {} {}'.format(\n            can_symlink_directories, can_symlink_files))\n\n    try:\n        # test that we can create junctions, even if symlinks are disabled\n        djunc = _win32_junction(dpath, join(tempdir, 'djunc'))\n        fjunc = _win32_junction(fpath, join(tempdir, 'fjunc.txt'))\n        if testing:\n            _win32_junction(broken_dpath, join(tempdir, 'broken_djunc'))\n            _win32_junction(broken_fpath, join(tempdir, 'broken_fjunc.txt'))\n        assert _win32_is_junction(djunc)\n        assert _win32_is_hardlinked(fpath, fjunc)\n    except Exception:\n        warnings.warn('We cannot create junctions either!')\n        raise\n\n    if testing:\n        # break the links\n        util_io.delete(broken_dpath)\n        util_io.delete(broken_fpath)\n\n        if verbose:\n            from ubelt import util_links\n            util_links._dirstats(tempdir)\n\n    try:\n        # Cleanup the test directory\n        util_io.delete(tempdir)\n    except Exception:\n        print('ERROR IN DELETE')\n        from ubelt import util_links\n        util_links._dirstats(tempdir)\n        raise\n\n    can_symlink = can_symlink_directories and can_symlink_files\n    __win32_can_symlink__ = can_symlink\n    if not can_symlink:\n        warnings.warn('Cannot make real symlink. Falling back to junction')\n\n    if verbose:\n        print('can_symlink = {!r}'.format(can_symlink))\n        print('__win32_can_symlink__ = {!r}'.format(__win32_can_symlink__))\n    return can_symlink", "response": "Return a new unicode object that can be used as a symbolic link to the current base."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _symlink(path, link, overwrite=0, verbose=0):\n    if exists(link) and not os.path.islink(link):\n        # On windows a broken link might still exist as a hard link or a\n        # junction. Overwrite it if it is a file and we cannot symlink.\n        # However, if it is a non-junction directory then do not overwrite\n        if verbose:\n            print('link location already exists')\n        is_junc = _win32_is_junction(link)\n        # NOTE:\n        # in python2 broken junctions are directories and exist\n        # in python3 broken junctions are directories and do not exist\n        if os.path.isdir(link):\n            if is_junc:\n                pointed = _win32_read_junction(link)\n                if path == pointed:\n                    if verbose:\n                        print('...and is a junction that points to the same place')\n                    return link\n                else:\n                    if verbose:\n                        if not exists(pointed):\n                            print('...and is a broken junction that points somewhere else')\n                        else:\n                            print('...and is a junction that points somewhere else')\n            else:\n                if verbose:\n                    print('...and is an existing real directory!')\n                raise IOError('Cannot overwrite a real directory')\n\n        elif os.path.isfile(link):\n            if _win32_is_hardlinked(link, path):\n                if verbose:\n                    print('...and is a hard link that points to the same place')\n                return link\n            else:\n                if verbose:\n                    print('...and is a hard link that points somewhere else')\n                if _win32_can_symlink():\n                    raise IOError('Cannot overwrite potentially real file if we can symlink')\n        if overwrite:\n            if verbose:\n                print('...overwriting')\n            util_io.delete(link, verbose > 1)\n        else:\n            if exists(link):\n                raise IOError('Link already exists')\n\n    _win32_symlink2(path, link, verbose=verbose)", "response": "Windows helper for ub.symlink"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _win32_symlink2(path, link, allow_fallback=True, verbose=0):\n    if _win32_can_symlink():\n        return _win32_symlink(path, link, verbose)\n    else:\n        return _win32_junction(path, link, verbose)", "response": "A symlink implementation that uses a real symlink if possible."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _win32_symlink(path, link, verbose=0):\n    from ubelt import util_cmd\n    if os.path.isdir(path):\n        # directory symbolic link\n        if verbose:\n            print('... as directory symlink')\n        command = 'mklink /D \"{}\" \"{}\"'.format(link, path)\n        # Using the win32 API seems to result in privilege errors\n        # but using shell commands does not have this problem. Weird.\n        # jwfs.symlink(path, link, target_is_directory=True)\n        # TODO: what do we need to do to use the windows api instead of shell?\n    else:\n        # file symbolic link\n        if verbose:\n            print('... as file symlink')\n        command = 'mklink \"{}\" \"{}\"'.format(link, path)\n\n    if command is not None:\n        info = util_cmd.cmd(command, shell=True)\n        if info['ret'] != 0:\n            from ubelt import util_format\n            permission_msg = 'You do not have sufficient privledges'\n            if permission_msg not in info['err']:\n                print('Failed command:')\n                print(info['command'])\n                print(util_format.repr2(info, nl=1))\n            raise OSError(str(info))\n    return link", "response": "Create a real symlink on Windows 10 and later."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndetermining if a path is a win32 junction", "response": "def _win32_is_junction(path):\n    \"\"\"\n    Determines if a path is a win32 junction\n\n    CommandLine:\n        python -m ubelt._win32_links _win32_is_junction\n\n    Example:\n        >>> # xdoc: +REQUIRES(WIN32)\n        >>> import ubelt as ub\n        >>> root = ub.ensure_app_cache_dir('ubelt', 'win32_junction')\n        >>> ub.delete(root)\n        >>> ub.ensuredir(root)\n        >>> dpath = join(root, 'dpath')\n        >>> djunc = join(root, 'djunc')\n        >>> ub.ensuredir(dpath)\n        >>> _win32_junction(dpath, djunc)\n        >>> assert _win32_is_junction(djunc) is True\n        >>> assert _win32_is_junction(dpath) is False\n        >>> assert _win32_is_junction('notafile') is False\n    \"\"\"\n    if not exists(path):\n        if os.path.isdir(path):\n            if not os.path.islink(path):\n                return True\n        return False\n    return jwfs.is_reparse_point(path) and not os.path.islink(path)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _win32_read_junction(path):\n    if not jwfs.is_reparse_point(path):\n        raise ValueError('not a junction')\n\n    # --- Older version based on using shell commands ---\n    # if not exists(path):\n    #     if six.PY2:\n    #         raise OSError('Cannot find path={}'.format(path))\n    #     else:\n    #         raise FileNotFoundError('Cannot find path={}'.format(path))\n    # target_name = os.path.basename(path)\n    # for type_or_size, name, pointed in _win32_dir(path, '*'):\n    #     if type_or_size == '<JUNCTION>' and name == target_name:\n    #         return pointed\n    # raise ValueError('not a junction')\n\n    # new version using the windows api\n    handle = jwfs.api.CreateFile(\n            path, 0, 0, None, jwfs.api.OPEN_EXISTING,\n            jwfs.api.FILE_FLAG_OPEN_REPARSE_POINT |\n            jwfs.api.FILE_FLAG_BACKUP_SEMANTICS,\n            None)\n\n    if handle == jwfs.api.INVALID_HANDLE_VALUE:\n        raise WindowsError()\n\n    res = jwfs.reparse.DeviceIoControl(\n            handle, jwfs.api.FSCTL_GET_REPARSE_POINT, None, 10240)\n\n    bytes = jwfs.create_string_buffer(res)\n    p_rdb = jwfs.cast(bytes, jwfs.POINTER(jwfs.api.REPARSE_DATA_BUFFER))\n    rdb = p_rdb.contents\n\n    if rdb.tag not in [2684354563, jwfs.api.IO_REPARSE_TAG_SYMLINK]:\n        raise RuntimeError(\n                \"Expected <2684354563 or 2684354572>, but got %d\" % rdb.tag)\n\n    jwfs.handle_nonzero_success(jwfs.api.CloseHandle(handle))\n    subname = rdb.get_substitute_name()\n    # probably has something to do with long paths, not sure\n    if subname.startswith('?\\\\'):\n        subname = subname[2:]\n    return subname", "response": "Return the location that the junction points at path raises ValueError if path is not a junction."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nremove all junctions in a directory", "response": "def _win32_rmtree(path, verbose=0):\n    \"\"\"\n    rmtree for win32 that treats junctions like directory symlinks.\n    The junction removal portion may not be safe on race conditions.\n\n    There is a known issue that prevents shutil.rmtree from\n    deleting directories with junctions.\n    https://bugs.python.org/issue31226\n    \"\"\"\n\n    # --- old version using the shell ---\n    # def _rmjunctions(root):\n    #     subdirs = []\n    #     for type_or_size, name, pointed in _win32_dir(root):\n    #         if type_or_size == '<DIR>':\n    #             subdirs.append(name)\n    #         elif type_or_size == '<JUNCTION>':\n    #             # remove any junctions as we encounter them\n    #             # os.unlink(join(root, name))\n    #             os.rmdir(join(root, name))\n    #     # recurse in all real directories\n    #     for name in subdirs:\n    #         _rmjunctions(join(root, name))\n\n    def _rmjunctions(root):\n        subdirs = []\n        for name in os.listdir(root):\n            current = join(root, name)\n            if os.path.isdir(current):\n                if _win32_is_junction(current):\n                    # remove any junctions as we encounter them\n                    os.rmdir(current)\n                elif not os.path.islink(current):\n                    subdirs.append(current)\n        # recurse in all real directories\n        for subdir in subdirs:\n            _rmjunctions(subdir)\n\n    if _win32_is_junction(path):\n        if verbose:\n            print('Deleting <JUNCTION> directory=\"{}\"'.format(path))\n        os.rmdir(path)\n    else:\n        if verbose:\n            print('Deleting directory=\"{}\"'.format(path))\n        # first remove all junctions\n        _rmjunctions(path)\n        # now we can rmtree as normal\n        import shutil\n        shutil.rmtree(path)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ntest if two hard links point to the same location.", "response": "def _win32_is_hardlinked(fpath1, fpath2):\n    \"\"\"\n    Test if two hard links point to the same location\n\n    CommandLine:\n        python -m ubelt._win32_links _win32_is_hardlinked\n\n    Example:\n        >>> # xdoc: +REQUIRES(WIN32)\n        >>> import ubelt as ub\n        >>> root = ub.ensure_app_cache_dir('ubelt', 'win32_hardlink')\n        >>> ub.delete(root)\n        >>> ub.ensuredir(root)\n        >>> fpath1 = join(root, 'fpath1')\n        >>> fpath2 = join(root, 'fpath2')\n        >>> ub.touch(fpath1)\n        >>> ub.touch(fpath2)\n        >>> fjunc1 = _win32_junction(fpath1, join(root, 'fjunc1'))\n        >>> fjunc2 = _win32_junction(fpath2, join(root, 'fjunc2'))\n        >>> assert _win32_is_hardlinked(fjunc1, fpath1)\n        >>> assert _win32_is_hardlinked(fjunc2, fpath2)\n        >>> assert not _win32_is_hardlinked(fjunc2, fpath1)\n        >>> assert not _win32_is_hardlinked(fjunc1, fpath2)\n    \"\"\"\n    # NOTE: jwf.samefile(fpath1, fpath2) seems to behave differently\n    def get_read_handle(fpath):\n        if os.path.isdir(fpath):\n            dwFlagsAndAttributes = jwfs.api.FILE_FLAG_BACKUP_SEMANTICS\n        else:\n            dwFlagsAndAttributes = 0\n        hFile = jwfs.api.CreateFile(fpath, jwfs.api.GENERIC_READ,\n                                    jwfs.api.FILE_SHARE_READ, None,\n                                    jwfs.api.OPEN_EXISTING,\n                                    dwFlagsAndAttributes, None)\n        return hFile\n\n    def get_unique_id(hFile):\n        info = jwfs.api.BY_HANDLE_FILE_INFORMATION()\n        res = jwfs.api.GetFileInformationByHandle(hFile, info)\n        jwfs.handle_nonzero_success(res)\n        unique_id = (info.volume_serial_number, info.file_index_high,\n                     info.file_index_low)\n        return unique_id\n\n    hFile1 = get_read_handle(fpath1)\n    hFile2 = get_read_handle(fpath2)\n    try:\n        are_equal = (get_unique_id(hFile1) == get_unique_id(hFile2))\n    except Exception:\n        raise\n    finally:\n        jwfs.api.CloseHandle(hFile1)\n        jwfs.api.CloseHandle(hFile2)\n    return are_equal"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nget information about a directory in Windows.", "response": "def _win32_dir(path, star=''):\n    \"\"\"\n    Using the windows cmd shell to get information about a directory\n    \"\"\"\n    from ubelt import util_cmd\n    import re\n    wrapper = 'cmd /S /C \"{}\"'  # the /S will preserve all inner quotes\n    command = 'dir /-C \"{}\"{}'.format(path, star)\n    wrapped = wrapper.format(command)\n    info = util_cmd.cmd(wrapped, shell=True)\n    if info['ret'] != 0:\n        from ubelt import util_format\n        print('Failed command:')\n        print(info['command'])\n        print(util_format.repr2(info, nl=1))\n        raise OSError(str(info))\n    # parse the output of dir to get some info\n    # Remove header and footer\n    lines = info['out'].split('\\n')[5:-3]\n    splitter = re.compile('( +)')\n    for line in lines:\n        parts = splitter.split(line)\n        date, sep, time, sep, ampm, sep, type_or_size, sep = parts[:8]\n        name = ''.join(parts[8:])\n        # if type is a junction then name will also contain the linked loc\n        if name == '.' or name == '..':\n            continue\n        if type_or_size in ['<JUNCTION>', '<SYMLINKD>', '<SYMLINK>']:\n            # colons cannot be in path names, so use that to find where\n            # the name ends\n            pos = name.find(':')\n            bpos = name[:pos].rfind('[')\n            name = name[:bpos - 1]\n            pointed = name[bpos + 1:-1]\n            yield type_or_size, name, pointed\n        else:\n            yield type_or_size, name, None"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_generator_doubling(config):\n    start = 1\n    if 'start' in config:\n        start = int(config['start'])\n\n    # We cannot simply use start as the variable, because of scoping\n    # limitations\n    def generator():\n        val = start\n        while(True):\n            yield val\n            val = val * 2\n    return generator()", "response": "Returns a generator that double with each value returned\nAttributeNames Config includes optional start value"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nparsing a contains validator which takes as the config a simple string to find", "response": "def parse(config):\n        \"\"\" Parse a contains validator, which takes as the config a simple string to find \"\"\"\n        if not isinstance(config, basestring):\n            raise TypeError(\"Contains input must be a simple string\")\n        validator = ContainsValidator()\n        validator.contains_string = config\n        return validator"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nretrieve the adjacency matrix from the nx. DiGraph or np. ndarray.", "response": "def retrieve_adjacency_matrix(graph, order_nodes=None, weight=False):\n    \"\"\"Retrieve the adjacency matrix from the nx.DiGraph or numpy array.\"\"\"\n    if isinstance(graph, np.ndarray):\n        return graph\n    elif isinstance(graph, nx.DiGraph):\n        if order_nodes is None:\n            order_nodes = graph.nodes()\n        if not weight:\n            return np.array(nx.adjacency_matrix(graph, order_nodes, weight=None).todense())\n        else:\n            return np.array(nx.adjacency_matrix(graph, order_nodes).todense())\n    else:\n        raise TypeError(\"Only networkx.DiGraph and np.ndarray (adjacency matrixes) are supported.\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef precision_recall(target, prediction, low_confidence_undirected=False):\n    true_labels = retrieve_adjacency_matrix(target)\n    pred = retrieve_adjacency_matrix(prediction, target.nodes()\n                                            if isinstance(target, nx.DiGraph) else None,\n                                            weight=True)\n    \n    if low_confidence_undirected:\n        # Take account of undirected edges by putting them with low confidence\n        pred[pred==pred.transpose()] *= min(min(pred[np.nonzero(pred)])*.5, .1)\n    precision, recall, _ = precision_recall_curve(\n        true_labels.ravel(), pred.ravel())\n    aupr = auc(recall, precision, reorder=True)\n\n    return aupr, list(zip(precision, recall))", "response": "r Compute precision - recall statistics for a set of target graphs and predictions."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef SHD(target, pred, double_for_anticausal=True):\n    true_labels = retrieve_adjacency_matrix(target)\n    predictions = retrieve_adjacency_matrix(pred, target.nodes() \n                                            if isinstance(target, nx.DiGraph) else None)\n    \n    diff = np.abs(true_labels - predictions)\n    if double_for_anticausal:\n        return np.sum(diff)\n    else:\n        diff = diff + diff.transpose()\n        diff[diff > 1] = 1  # Ignoring the double edges.\n        return np.sum(diff)/2", "response": "r Compute the Structural Hamming Distance between two graphs."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncomputing the Strutural Intervention Distance for a given target and prediction.", "response": "def SID(target, pred):\n    \"\"\"Compute the Strutural Intervention Distance.\n    \n    [R wrapper] The Structural Intervention Distance (SID) is a new distance\n    for graphs introduced by Peters and B\u00fchlmann (2013). This distance was\n    created to account for the shortcomings of the SHD metric for a causal \n    sense.\n    It consists in computing the path between all the pairs of variables, and\n    checks if the causal relationship between the variables is respected.\n    The given graphs have to be DAGs for the SID metric to make sense.\n\n    Args:\n        target (numpy.ndarray or networkx.DiGraph): Target graph, must be of \n            ones and zeros, and instance of either numpy.ndarray or \n            networkx.DiGraph. Must be a DAG.\n\n        prediction (numpy.ndarray or networkx.DiGraph): Prediction made by the\n            algorithm to evaluate. Must be a DAG.\n \n    Returns:\n        int: Structural Intervention Distance. \n\n            The value tends to zero as the graphs tends to be identical.\n        \n    .. note::\n        Ref: Structural Intervention Distance (SID) for Evaluating Causal Graphs,\n        Jonas Peters, Peter B\u00fchlmann: https://arxiv.org/abs/1306.1043\n    \n    Examples:\n        >>> from numpy.random import randint\n        >>> tar = np.triu(randint(2, size=(10, 10))) \n        >>> pred = np.triu(randint(2, size=(10, 10)))\n        >>> SID(tar, pred) \n   \"\"\"\n    if not RPackages.SID:\n        raise ImportError(\"SID R package is not available. Please check your installation.\")\n\n    true_labels = retrieve_adjacency_matrix(target)\n    predictions = retrieve_adjacency_matrix(pred, target.nodes() \n                                            if isinstance(target, nx.DiGraph) else None)\n    \n    os.makedirs('/tmp/cdt_SID/')\n\n    def retrieve_result():\n        return np.loadtxt('/tmp/cdt_SID/result.csv')\n\n    try:\n        np.savetxt('/tmp/cdt_SID/target.csv', true_labels, delimiter=',')\n        np.savetxt('/tmp/cdt_SID/pred.csv', predictions, delimiter=',')\n        sid_score = launch_R_script(\"{}/R_templates/sid.R\".format(os.path.dirname(os.path.realpath(__file__))),\n                                    {\"{target}\": '/tmp/cdt_SID/target.csv',\n                                     \"{prediction}\": '/tmp/cdt_SID/pred.csv',\n                                     \"{result}\": '/tmp/cdt_SID/result.csv'},\n                                    output_function=retrieve_result)\n    # Cleanup\n    except Exception as e:\n        rmtree('/tmp/cdt_SID')\n        raise e\n    except KeyboardInterrupt:\n        rmtree('/tmp/cdt_SID/')\n        raise KeyboardInterrupt\n\n    rmtree('/tmp/cdt_SID')\n    return sid_score"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate a networkx. DiGraph from the data produced by the CCDR algorithm.", "response": "def create_graph_from_data(self, data, **kwargs):\n        \"\"\"Apply causal discovery on observational data using CCDr.\n\n        Args:\n            data (pandas.DataFrame): DataFrame containing the data\n\n        Returns:\n            networkx.DiGraph: Solution given by the CCDR algorithm.\n        \"\"\"\n        # Building setup w/ arguments.\n        self.arguments['{VERBOSE}'] = str(self.verbose).upper()\n        results = self._run_ccdr(data, verbose=self.verbose)\n        return nx.relabel_nodes(nx.DiGraph(results),\n                                {idx: i for idx, i in enumerate(data.columns)})"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef init_variables(self, verbose=False):\n        for j in range(1, self.nodes):\n            nb_parents = np.random.randint(0, min([self.parents_max, j])+1)\n            for i in np.random.choice(range(0, j), nb_parents, replace=False):\n                self.adjacency_matrix[i, j] = 1\n\n        try:\n            self.g = nx.DiGraph(self.adjacency_matrix)\n            assert not list(nx.simple_cycles(self.g))\n\n        except AssertionError:\n            if verbose:\n                print(\"Regenerating, graph non valid...\")\n            self.init_variables()\n\n        # Mechanisms\n        self.cfunctions = [self.mechanism(int(sum(self.adjacency_matrix[:, i])),\n                                          self.points, self.noise, noise_coeff=self.noise_coeff)\n                           if sum(self.adjacency_matrix[:, i])\n                           else self.initial_generator for i in range(self.nodes)]", "response": "Redefine the causes of the graph."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating data from an FCM containing cycles.", "response": "def generate(self, rescale=True):\n        \"\"\"Generate data from an FCM containing cycles.\"\"\"\n        if self.cfunctions is None:\n            self.init_variables()\n\n        for i in nx.topological_sort(self.g):\n            # Root cause\n\n            if not sum(self.adjacency_matrix[:, i]):\n                self.data['V{}'.format(i)] = self.cfunctions[i](self.points)\n            # Generating causes\n            else:\n                self.data['V{}'.format(i)] = self.cfunctions[i](self.data.iloc[:, self.adjacency_matrix[:, i].nonzero()[0]].values)\n            if rescale:\n                self.data['V{}'.format(i)] = scale(self.data['V{}'.format(i)].values)\n\n        return self.g, self.data"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef to_csv(self, fname_radical, **kwargs):\n        if self.data is not None:\n            self.data.to_csv(fname_radical+'_data.csv', index=False, **kwargs)\n            pd.DataFrame(self.adjacency_matrix).to_csv(fname_radical \\\n                                                       + '_target.csv',\n                                                       index=False, **kwargs)\n\n        else:\n            raise ValueError(\"Graph has not yet been generated. \\\n                              Use self.generate() to do so.\")", "response": "Saves data to the csv format by default in two separate files."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef launch_R_script(template, arguments, output_function=None,\n                    verbose=True, debug=False):\n    \"\"\"Launch an R script, starting from a template and replacing text in file\n    before execution.\n\n    Args:\n        template (str): path to the template of the R script\n        arguments (dict): Arguments that modify the template's placeholders\n            with arguments\n        output_function (function): Function to execute **after** the execution\n            of the R script, and its output is returned by this function. Used\n            traditionally as a function to retrieve the results of the\n            execution.\n        verbose (bool): Sets the verbosity of the R subprocess.\n        debug (bool): If True, the generated scripts are not deleted.\n\n    Return:\n        Returns the output of the ``output_function`` if not `None`\n        else `True` or `False` depending on whether the execution was\n        successful.\n    \"\"\"\n    id = str(uuid.uuid4())\n    os.makedirs('/tmp/cdt_R_script_' + id + '/')\n    try:\n        scriptpath = '/tmp/cdt_R_script_' + id + '/instance_{}'.format(os.path.basename(template))\n        copy(template, scriptpath)\n\n        with fileinput.FileInput(scriptpath, inplace=True) as file:\n            for line in file:\n                mline = line\n                for elt in arguments:\n                    mline = mline.replace(elt, arguments[elt])\n                print(mline, end='')\n\n        if output_function is None:\n            output = subprocess.call(\"Rscript --vanilla {}\".format(scriptpath), shell=True,\n                                     stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n        else:\n            if verbose:\n                process = subprocess.Popen(\"Rscript --vanilla {}\".format(scriptpath), shell=True)\n            else:\n                process = subprocess.Popen(\"Rscript --vanilla {}\".format(scriptpath), shell=True,\n                                           stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n            process.wait()\n            output = output_function()\n\n    # Cleaning up\n    except Exception as e:\n        if not debug:\n            rmtree('/tmp/cdt_R_script_' + id + '/')\n        raise e\n    except KeyboardInterrupt:\n        if not debug:\n            rmtree('/tmp/cdt_R_script_' + id + '/')\n        raise KeyboardInterrupt\n    if not debug:\n        rmtree('/tmp/cdt_R_script_' + id + '/')\n    return output", "response": "Launch an R script in a virtual machine and replacing text in file\nracket with arguments."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nexecuting a subprocess to check the package s availability.", "response": "def check_R_package(self, package):\n        \"\"\"Execute a subprocess to check the package's availability.\n\n        Args:\n            package (str): Name of the package to be tested.\n\n        Returns:\n            bool: `True` if the package is available, `False` otherwise\n        \"\"\"\n        test_package = not bool(launch_R_script(\"{}/R_templates/test_import.R\".format(os.path.dirname(os.path.realpath(__file__))),                                      {\"{package}\": package}, verbose=True))\n        return test_package"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nbinning variables w / normalization.", "response": "def bin_variable(var, bins='fd'):  # bin with normalization\n    \"\"\"Bin variables w/ normalization.\"\"\"\n    var = np.array(var).astype(np.float)\n    var = (var - np.mean(var)) / np.std(var)\n    var = np.digitize(var, np.histogram(var, bins=bins)[1])\n\n    return var"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef predict(self, a, b, **kwargs):\n        binning_alg = kwargs.get('bins', 'fd')\n        return metrics.adjusted_mutual_info_score(bin_variable(a, bins=binning_alg),\n                                                  bin_variable(b, bins=binning_alg))", "response": "Perform the independence test."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef predict(self, df_data, graph=None, **kwargs):\n        if graph is None:\n            return self.create_graph_from_data(df_data, **kwargs)\n        elif isinstance(graph, nx.DiGraph):\n            return self.orient_directed_graph(df_data, graph, **kwargs)\n        elif isinstance(graph, nx.Graph):\n            return self.orient_undirected_graph(df_data, graph, **kwargs)\n        else:\n            print('Unknown Graph type')\n            raise ValueError", "response": "Orient a causal graph using the method defined by the arguments."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef graph_evaluation(data, adj_matrix, gpu=None, gpu_id=0, **kwargs):\n    gpu = SETTINGS.get_default(gpu=gpu)\n    device = 'cuda:{}'.format(gpu_id) if gpu else 'cpu'\n    obs = th.FloatTensor(data).to(device)\n    cgnn = CGNN_model(adj_matrix, data.shape[0], gpu_id=gpu_id, **kwargs).to(device)\n    cgnn.reset_parameters()\n    return cgnn.run(obs, **kwargs)", "response": "Evaluate a graph taking account of the hardware."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef parallel_graph_evaluation(data, adj_matrix, nb_runs=16,\n                              nb_jobs=None, **kwargs):\n    \"\"\"Parallelize the various runs of CGNN to evaluate a graph.\"\"\"\n    nb_jobs = SETTINGS.get_default(nb_jobs=nb_jobs)\n    if nb_runs == 1:\n        return graph_evaluation(data, adj_matrix, **kwargs)\n    else:\n        output = Parallel(n_jobs=nb_jobs)(delayed(graph_evaluation)(data, adj_matrix,\n                                          idx=run, gpu_id=run % SETTINGS.GPU,\n                                          **kwargs) for run in range(nb_runs))\n        return np.mean(output)", "response": "Parallelize the various runs of CGNN to evaluate a graph."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef hill_climbing(data, graph, **kwargs):\n    nodelist = list(data.columns)\n    data = scale(data.values).astype('float32')\n    tested_candidates = [nx.adj_matrix(graph, nodelist=nodelist, weight=None)]\n    best_score = parallel_graph_evaluation(data, tested_candidates[0].todense(), ** kwargs)\n    best_candidate = graph\n    can_improve = True\n    while can_improve:\n        can_improve = False\n        for (i, j) in best_candidate.edges():\n            test_graph = deepcopy(best_candidate)\n            test_graph.add_edge(j, i, weight=test_graph[i][j]['weight'])\n            test_graph.remove_edge(i, j)\n            tadjmat = nx.adj_matrix(test_graph, nodelist=nodelist, weight=None)\n            if (nx.is_directed_acyclic_graph(test_graph) and not any([(tadjmat != cand).nnz ==\n                                                                      0 for cand in tested_candidates])):\n                tested_candidates.append(tadjmat)\n                score = parallel_graph_evaluation(data, tadjmat.todense(), **kwargs)\n                if score < best_score:\n                    can_improve = True\n                    best_candidate = test_graph\n                    best_score = score\n                    break\n    return best_candidate", "response": "Hill Climbing optimization: a greedy exploration algorithm."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngenerating according to the topological order of the graph.", "response": "def forward(self):\n        \"\"\"Generate according to the topological order of the graph.\"\"\"\n        self.noise.data.normal_()\n        if not self.confounding:\n            for i in self.topological_order:\n                self.generated[i] = self.blocks[i](th.cat([v for c in [\n                                                   [self.generated[j] for j in np.nonzero(self.adjacency_matrix[:, i])[0]],\n                                                   [self.noise[:, [i]]]] for v in c], 1))\n        else:\n            for i in self.topological_order:\n                self.generated[i] = self.blocks[i](th.cat([v for c in [\n                                                   [self.generated[j] for j in np.nonzero(self.adjacency_matrix[:, i])[0]],\n                                                   [self.corr_noise[min(i, j), max(i, j)] for j in np.nonzero(self.i_adj_matrix[:, i])[0]]\n                                                   [self.noise[:, [i]]]] for v in c], 1))\n        return th.cat(self.generated, 1)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nrun the CGNN on a given graph.", "response": "def run(self, data, train_epochs=1000, test_epochs=1000, verbose=None,\n            idx=0, lr=0.01, **kwargs):\n        \"\"\"Run the CGNN on a given graph.\"\"\"\n        verbose = SETTINGS.get_default(verbose=verbose)\n        optim = th.optim.Adam(self.parameters(), lr=lr)\n        self.score.zero_()\n        with trange(train_epochs + test_epochs, disable=not verbose) as t:\n            for epoch in t:\n                optim.zero_grad()\n                generated_data = self.forward()\n                mmd = self.criterion(generated_data, data)\n                if not epoch % 200:\n                    t.set_postfix(idx=idx, epoch=epoch, loss=mmd.item())\n                mmd.backward()\n                optim.step()\n                if epoch >= test_epochs:\n                    self.score.add_(mmd.data)\n\n        return self.score.cpu().numpy() / test_epochs"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nuses CGNN to create a graph from a pandas. DataFrame.", "response": "def create_graph_from_data(self, data):\n        \"\"\"Use CGNN to create a graph from scratch. All the possible structures\n        are tested, which leads to a super exponential complexity. It would be\n        preferable to start from a graph skeleton for large graphs.\n\n        Args:\n            data (pandas.DataFrame): Observational data on which causal\n               discovery has to be performed.\n        Returns:\n            networkx.DiGraph: Solution given by CGNN.\n\n        \"\"\"\n        warnings.warn(\"An exhaustive search of the causal structure of CGNN without\"\n                      \" skeleton is super-exponential in the number of variables.\")\n\n        # Building all possible candidates:\n        nb_vars = len(list(data.columns))\n        data = scale(data.values).astype('float32')\n\n        candidates = [np.reshape(np.array(i), (nb_vars, nb_vars)) for i in itertools.product([0, 1], repeat=nb_vars*nb_vars)\n                      if (np.trace(np.reshape(np.array(i), (nb_vars, nb_vars))) == 0\n                          and nx.is_directed_acyclic_graph(nx.DiGraph(np.reshape(np.array(i), (nb_vars, nb_vars)))))]\n\n        warnings.warn(\"A total of {} graphs will be evaluated.\".format(len(candidates)))\n        scores = [parallel_graph_evaluation(data, i, nh=self.nh, nb_runs=self.nb_runs, gpu=self.gpu,\n                                            nb_jobs=self.nb_jobs, lr=self.lr, train_epochs=self.train_epochs,\n                                            test_epochs=self.test_epochs, verbose=self.verbose) for i in candidates]\n        final_candidate = candidates[scores.index(min(scores))]\n        output = np.zeros(final_candidate.shape)\n\n        # Retrieve the confidence score on each edge.\n        for (i, j), x in np.ndenumerate(final_candidate):\n            if x > 0:\n                cand = final_candidate\n                cand[i, j] = 0\n                output[i, j] = min(scores) - scores[candidates.index(cand)]\n\n        return nx.DiGraph(candidates[output],\n                          {idx: i for idx, i in enumerate(data.columns)})"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nmodify and improve a directed acyclic graph solution using CGNN.", "response": "def orient_directed_graph(self, data, dag, alg='HC'):\n        \"\"\"Modify and improve a directed acyclic graph solution using CGNN.\n\n        Args:\n            data (pandas.DataFrame): Observational data on which causal\n               discovery has to be performed.\n            dag (nx.DiGraph): Graph that provides the initial solution,\n               on which the CGNN algorithm will be applied.\n            alg (str): Exploration heuristic to use, among [\"HC\", \"HCr\",\n               \"tabu\", \"EHC\"]\n        Returns:\n            networkx.DiGraph: Solution given by CGNN.\n       \n        \"\"\"\n        alg_dic = {'HC': hill_climbing, 'HCr': hill_climbing_with_removal,\n                   'tabu': tabu_search, 'EHC': exploratory_hill_climbing}\n\n        return alg_dic[alg](data, dag, nh=self.nh, nb_runs=self.nb_runs, gpu=self.gpu,\n                            nb_jobs=self.nb_jobs, lr=self.lr, train_epochs=self.train_epochs,\n                            test_epochs=self.test_epochs, verbose=self.verbose)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef orient_undirected_graph(self, data, umg, alg='HC'):\n        warnings.warn(\"The pairwise GNN model is computed on each edge of the UMG \"\n                      \"to initialize the model and start CGNN with a DAG\")\n        gnn = GNN(nh=self.nh, lr=self.lr)\n\n        og = gnn.orient_graph(data, umg, nb_runs=self.nb_runs, nb_max_runs=self.nb_runs,\n                              nb_jobs=self.nb_jobs, train_epochs=self.train_epochs,\n                              test_epochs=self.test_epochs, verbose=self.verbose, gpu=self.gpu)  # Pairwise method\n        # print(nx.adj_matrix(og).todense().shape)\n        # print(list(og.edges()))\n        dag = dagify_min_edge(og)\n        # print(nx.adj_matrix(dag).todense().shape)\n\n        return self.orient_directed_graph(data, dag, alg=alg)", "response": "Orient the undirected graph using GNN and apply CGNN to improve the graph."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nevaluating the entropy of the input variable.", "response": "def eval_entropy(x):\n    \"\"\"Evaluate the entropy of the input variable.\n\n    :param x: input variable 1D\n    :return: entropy of x\n    \"\"\"\n    hx = 0.\n    sx = sorted(x)\n    for i, j in zip(sx[:-1], sx[1:]):\n        delta = j-i\n        if bool(delta):\n            hx += np.log(np.abs(delta))\n    hx = hx / (len(x) - 1) + psi(len(x)) - psi(1)\n\n    return hx"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef integral_approx_estimator(x, y):\n    a, b = (0., 0.)\n    x = np.array(x)\n    y = np.array(y)\n    idx, idy = (np.argsort(x), np.argsort(y))\n\n    for x1, x2, y1, y2 in zip(x[[idx]][:-1], x[[idx]][1:], y[[idx]][:-1], y[[idx]][1:]):\n        if x1 != x2 and y1 != y2:\n            a = a + np.log(np.abs((y2 - y1) / (x2 - x1)))\n\n    for x1, x2, y1, y2 in zip(x[[idy]][:-1], x[[idy]][1:], y[[idy]][:-1], y[[idy]][1:]):\n        if x1 != x2 and y1 != y2:\n            b = b + np.log(np.abs((x2 - x1) / (y2 - y1)))\n\n    return (a - b)/len(x)", "response": "Integral approximation estimator for causal inference."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nevaluate a pair using the IGCI model.", "response": "def predict_proba(self, a, b, **kwargs):\n        \"\"\"Evaluate a pair using the IGCI model.\n\n        :param a: Input variable 1D\n        :param b: Input variable 1D\n        :param kwargs: {refMeasure: Scaling method (gaussian, integral or None),\n                        estimator: method used to evaluate the pairs (entropy or integral)}\n        :return: Return value of the IGCI model >0 if a->b otherwise if return <0\n        \"\"\"\n        estimators = {'entropy': lambda x, y: eval_entropy(y) - eval_entropy(x), 'integral': integral_approx_estimator}\n        ref_measures = {'gaussian': lambda x: standard_scale.fit_transform(x.reshape((-1, 1))),\n                        'uniform': lambda x: min_max_scale.fit_transform(x.reshape((-1, 1))), 'None': lambda x: x}\n\n        ref_measure = ref_measures[kwargs.get('refMeasure', 'gaussian')]\n        estimator = estimators[kwargs.get('estimator', 'entropy')]\n\n        a = ref_measure(a)\n        b = ref_measure(b)\n\n        return estimator(a, b)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef featurize_row(self, x, y):\n        x = x.ravel()\n        y = y.ravel()\n        b = np.ones(x.shape)\n        dx = np.cos(np.dot(self.W2, np.vstack((x, b)))).mean(1)\n        dy = np.cos(np.dot(self.W2, np.vstack((y, b)))).mean(1)\n        if(sum(dx) > sum(dy)):\n            return np.hstack((dx, dy,\n                              np.cos(np.dot(self.W, np.vstack((x, y, b)))).mean(1)))\n        else:\n            return np.hstack((dx, dy,\n                              np.cos(np.dot(self.W, np.vstack((y, x, b)))).mean(1)))", "response": "This function projects the causal pair to the RKHS using the sampled kernel approximation."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef fit(self, x, y):\n        train = np.vstack((np.array([self.featurize_row(row.iloc[0],\n                                                        row.iloc[1]) for idx, row in x.iterrows()]),\n                           np.array([self.featurize_row(row.iloc[1],\n                                                        row.iloc[0]) for idx, row in x.iterrows()])))\n        labels = np.vstack((y, -y)).ravel()\n        verbose = 1 if self.verbose else 0\n        self.clf = CLF(verbose=verbose,\n                       min_samples_leaf=self.L,\n                       n_estimators=self.E,\n                       max_depth=self.max_depth,\n                       n_jobs=self.n_jobs).fit(train, labels)", "response": "Train the model.\n\n        Args:\n            x_tr (pd.DataFrame): CEPC format dataframe containing the pairs\n            y_tr (pd.DataFrame or np.ndarray): labels associated to the pairs"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef predict_proba(self, x, y=None, **kwargs):\n        if self.clf is None:\n            raise ValueError(\"Model has to be trained before making predictions.\")\n        if x is pandas.Series:\n            input_ = self.featurize_row(x.iloc[0], x.iloc[1]).reshape((1, -1))\n        elif x is pandas.DataFrame:\n            input_ = np.array([self.featurize_row(x.iloc[0], x.iloc[1]) for row in x])\n        elif y is not None:\n            input_ = self.featurize_row(x, y).reshape((1, -1))\n        else:\n            raise TypeError(\"DataType not understood.\")\n        return self.clf.predict(input_)", "response": "Predict the causal score using a trained RCC model."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef predict_features(self, df_features, df_target, nh=20, idx=0, dropout=0.,\n                         activation_function=th.nn.ReLU, lr=0.01, l1=0.1,  batch_size=-1,\n                         train_epochs=1000, test_epochs=1000, device=None,\n                         verbose=None, nb_runs=3):\n        \"\"\"For one variable, predict its neighbours.\n\n        Args:\n            df_features (pandas.DataFrame):\n            df_target (pandas.Series):\n            nh (int): number of hidden units\n            idx (int): (optional) for printing purposes\n            dropout (float): probability of dropout (between 0 and 1)\n            activation_function (torch.nn.Module): activation function of the NN\n            lr (float): learning rate of Adam\n            l1 (float): L1 penalization coefficient\n            batch_size (int): batch size, defaults to full-batch\n            train_epochs (int): number of train epochs\n            test_epochs (int): number of test epochs\n            device (str): cuda or cpu device (defaults to ``cdt.SETTINGS.default_device``)\n            verbose (bool): verbosity (defaults to ``cdt.SETTINGS.verbose``)\n            nb_runs (int): number of bootstrap runs\n\n        Returns:\n            list: scores of each feature relatively to the target\n\n        \"\"\"\n        device, verbose = SETTINGS.get_default(('device', device), ('verbose', verbose))\n        x = th.FloatTensor(scale(df_features.values)).to(device)\n        y = th.FloatTensor(scale(df_target.values)).to(device)\n        out = []\n        for i in range(nb_runs):\n            model = FSGNN_model([x.size()[1] + 1, nh, 1],\n                                dropout=dropout,\n                                activation_function=activation_function).to(device)\n\n            out.append(model.train(x, y, lr=0.01, l1=0.1, batch_size=-1,\n                                   train_epochs=train_epochs, test_epochs=test_epochs,\n                                   device=device, verbose=verbose))\n        return list(np.mean(np.array(out), axis=0))", "response": "For one variable predict features and target."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef predict_undirected_graph(self, data):\n        graph = Graph()\n\n        for idx_i, i in enumerate(data.columns):\n            for idx_j, j in enumerate(data.columns[idx_i+1:]):\n                score = self.predict(data[i].values, data[j].values)\n                if abs(score) > 0.001:\n                    graph.add_edge(i, j, weight=score)\n\n        return graph", "response": "Predict a skeleton using a pairwise independence criterion."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nwrap around self. predict_features to predict the features relatively to the target variable.", "response": "def run_feature_selection(self, df_data, target, idx=0, **kwargs):\n        \"\"\"Run feature selection for one node: wrapper around\n        ``self.predict_features``.\n\n        Args:\n            df_data (pandas.DataFrame): All the observational data\n            target (str): Name of the target variable\n            idx (int): (optional) For printing purposes\n\n        Returns:\n            list: scores of each feature relatively to the target\n        \"\"\"\n        list_features = list(df_data.columns.values)\n        list_features.remove(target)\n        df_target = pd.DataFrame(df_data[target], columns=[target])\n        df_features = df_data[list_features]\n\n        return self.predict_features(df_features, df_target, idx=idx, **kwargs)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\npredicts the skeleton of the graph from the raw data.", "response": "def predict(self, df_data, threshold=0.05, **kwargs):\n        \"\"\"Predict the skeleton of the graph from raw data.\n\n        Returns iteratively the feature selection algorithm on each node.\n\n        Args:\n            df_data (pandas.DataFrame): data to construct a graph from\n            threshold (float): cutoff value for feature selection scores\n            kwargs (dict): additional arguments for algorithms\n\n        Returns:\n            networkx.Graph: predicted skeleton of the graph.\n        \"\"\"\n        nb_jobs = kwargs.get(\"nb_jobs\", SETTINGS.NB_JOBS)\n        list_nodes = list(df_data.columns.values)\n        if nb_jobs != 1:\n            result_feature_selection = Parallel(n_jobs=nb_jobs)(delayed(self.run_feature_selection)\n                                                                (df_data, node, idx, **kwargs)\n                                                                for idx, node in enumerate(list_nodes))\n        else:\n            result_feature_selection = [self.run_feature_selection(df_data, node, idx, **kwargs) for idx, node in enumerate(list_nodes)]\n        for idx, i in enumerate(result_feature_selection):\n            try:\n                i.insert(idx, 0)\n            except AttributeError:  # if results are numpy arrays\n                result_feature_selection[idx] = np.insert(i, idx, 0)\n        matrix_results = np.array(result_feature_selection)\n        matrix_results *= matrix_results.transpose()\n        np.fill_diagonal(matrix_results, 0)\n        matrix_results /= 2\n\n        graph = nx.Graph()\n\n        for (i, j), x in np.ndenumerate(matrix_results):\n            if matrix_results[i, j] > threshold:\n                graph.add_edge(list_nodes[i], list_nodes[j],\n                               weight=matrix_results[i, j])\n        for node in list_nodes:\n            if node not in graph.nodes():\n                graph.add_node(node)\n        return graph"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nrun GIES on an undirected graph.", "response": "def orient_undirected_graph(self, data, graph):\n        \"\"\"Run GIES on an undirected graph.\n\n        Args:\n            data (pandas.DataFrame): DataFrame containing the data\n            graph (networkx.Graph): Skeleton of the graph to orient\n\n        Returns:\n            networkx.DiGraph: Solution given by the GIES algorithm.\n\n        \"\"\"\n        # Building setup w/ arguments.\n        self.arguments['{VERBOSE}'] = str(self.verbose).upper()\n        self.arguments['{SCORE}'] = self.scores[self.score]\n\n        fe = DataFrame(nx.adj_matrix(graph, weight=None).todense())\n        fg = DataFrame(1 - fe.values)\n\n        results = self._run_gies(data, fixedGaps=fg, verbose=self.verbose)\n\n        return nx.relabel_nodes(nx.DiGraph(results),\n                                {idx: i for idx, i in enumerate(data.columns)})"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef create_graph_from_data(self, data):\n        # Building setup w/ arguments.\n        self.arguments['{SCORE}'] = self.scores[self.score]\n        self.arguments['{VERBOSE}'] = str(self.verbose).upper()\n\n        results = self._run_gies(data, verbose=self.verbose)\n\n        return nx.relabel_nodes(nx.DiGraph(results),\n                                {idx: i for idx, i in enumerate(data.columns)})", "response": "Run the GIES algorithm and return a networkx. DiGraph with the results."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _run_gies(self, data, fixedGaps=None, verbose=True):\n        # Run gies\n        id = str(uuid.uuid4())\n        os.makedirs('/tmp/cdt_gies' + id + '/')\n        self.arguments['{FOLDER}'] = '/tmp/cdt_gies' + id + '/'\n\n        def retrieve_result():\n            return read_csv('/tmp/cdt_gies' + id + '/result.csv', delimiter=',').values\n\n        try:\n            data.to_csv('/tmp/cdt_gies' + id + '/data.csv', header=False, index=False)\n            if fixedGaps is not None:\n                fixedGaps.to_csv('/tmp/cdt_gies' + id + '/fixedgaps.csv', index=False, header=False)\n                self.arguments['{SKELETON}'] = 'TRUE'\n            else:\n                self.arguments['{SKELETON}'] = 'FALSE'\n\n            gies_result = launch_R_script(\"{}/R_templates/gies.R\".format(os.path.dirname(os.path.realpath(__file__))),\n                                          self.arguments, output_function=retrieve_result, verbose=verbose)\n        # Cleanup\n        except Exception as e:\n            rmtree('/tmp/cdt_gies' + id + '')\n            raise e\n        except KeyboardInterrupt:\n            rmtree('/tmp/cdt_gies' + id + '/')\n            raise KeyboardInterrupt\n        rmtree('/tmp/cdt_gies' + id + '')\n        return gies_result", "response": "Run GIES with all arguments."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nplotting SAM's various losses.", "response": "def plot_curves(i_batch, adv_loss, gen_loss, l1_reg, cols):\n    \"\"\"Plot SAM's various losses.\"\"\"\n    from matplotlib import pyplot as plt\n    if i_batch == 0:\n        try:\n            ax.clear()\n            ax.plot(range(len(adv_plt)), adv_plt, \"r-\",\n                    linewidth=1.5, markersize=4,\n                    label=\"Discriminator\")\n            ax.plot(range(len(adv_plt)), gen_plt, \"g-\", linewidth=1.5,\n                    markersize=4, label=\"Generators\")\n            ax.plot(range(len(adv_plt)), l1_plt, \"b-\",\n                    linewidth=1.5, markersize=4,\n                    label=\"L1-Regularization\")\n            plt.legend()\n\n            adv_plt.append(adv_loss.cpu().data[0])\n            gen_plt.append(gen_loss.cpu().data[0] / cols)\n            l1_plt.append(l1_reg.cpu().data[0])\n\n            plt.pause(0.0001)\n\n        except NameError:\n            plt.ion()\n            fig, ax = plt.figure()\n            plt.xlabel(\"Epoch\")\n            plt.ylabel(\"Losses\")\n\n            plt.pause(0.0001)\n\n            adv_plt = [adv_loss.cpu().data[0]]\n            gen_plt = [gen_loss.cpu().data[0] / cols]\n            l1_plt = [l1_reg.cpu().data[0]]\n\n    else:\n        adv_plt.append(adv_loss.cpu().data[0])\n        gen_plt.append(gen_loss.cpu().data[0] / cols)\n        l1_plt.append(l1_reg.cpu().data[0])"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef plot_gen(epoch, batch, generated_variables, pairs_to_plot=[[0, 1]]):\n    from matplotlib import pyplot as plt\n    if epoch == 0:\n        plt.ion()\n    plt.clf()\n    for (i, j) in pairs_to_plot:\n\n        plt.scatter(generated_variables[i].data.cpu().numpy(\n        ), batch.data.cpu().numpy()[:, j], label=\"Y -> X\")\n        plt.scatter(batch.data.cpu().numpy()[\n            :, i], generated_variables[j].data.cpu().numpy(), label=\"X -> Y\")\n\n        plt.scatter(batch.data.cpu().numpy()[:, i], batch.data.cpu().numpy()[\n            :, j], label=\"original data\")\n        plt.legend()\n\n    plt.pause(0.01)", "response": "Plot generated pairs of variables."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef run_SAM(df_data, skeleton=None, **kwargs):\n    gpu = kwargs.get('gpu', False)\n    gpu_no = kwargs.get('gpu_no', 0)\n\n    train_epochs = kwargs.get('train_epochs', 1000)\n    test_epochs = kwargs.get('test_epochs', 1000)\n    batch_size = kwargs.get('batch_size', -1)\n\n    lr_gen = kwargs.get('lr_gen', 0.1)\n    lr_disc = kwargs.get('lr_disc', lr_gen)\n    verbose = kwargs.get('verbose', True)\n    regul_param = kwargs.get('regul_param', 0.1)\n    dnh = kwargs.get('dnh', None)\n    plot = kwargs.get(\"plot\", False)\n    plot_generated_pair = kwargs.get(\"plot_generated_pair\", False)\n\n    d_str = \"Epoch: {} -- Disc: {} -- Gen: {} -- L1: {}\"\n    try:\n        list_nodes = list(df_data.columns)\n        df_data = (df_data[list_nodes]).values\n    except AttributeError:\n        list_nodes = list(range(df_data.shape[1]))\n    data = df_data.astype('float32')\n    data = th.from_numpy(data)\n    if batch_size == -1:\n        batch_size = data.shape[0]\n    rows, cols = data.size()\n\n    # Get the list of indexes to ignore\n    if skeleton is not None:\n        zero_components = [[] for i in range(cols)]\n        skel = nx.adj_matrix(skeleton, weight=None)\n        for i, j in zip(*((1-skel).nonzero())):\n            zero_components[j].append(i)\n    else:\n        zero_components = [[i] for i in range(cols)]\n    sam = SAM_generators((rows, cols), zero_components, batch_norm=True, **kwargs)\n\n    # Begin UGLY\n    activation_function = kwargs.get('activation_function', th.nn.Tanh)\n    try:\n        del kwargs[\"activation_function\"]\n    except KeyError:\n        pass\n    discriminator_sam = SAM_discriminator(\n        [cols, dnh, dnh, 1], batch_norm=True,\n        activation_function=th.nn.LeakyReLU,\n        activation_argument=0.2, **kwargs)\n    kwargs[\"activation_function\"] = activation_function\n    # End of UGLY\n\n    if gpu:\n        sam = sam.cuda(gpu_no)\n        discriminator_sam = discriminator_sam.cuda(gpu_no)\n        data = data.cuda(gpu_no)\n\n    # Select parameters to optimize : ignore the non connected nodes\n    criterion = th.nn.BCEWithLogitsLoss()\n    g_optimizer = th.optim.Adam(sam.parameters(), lr=lr_gen)\n    d_optimizer = th.optim.Adam(\n        discriminator_sam.parameters(), lr=lr_disc)\n\n    true_variable = Variable(\n        th.ones(batch_size, 1), requires_grad=False)\n    false_variable = Variable(\n        th.zeros(batch_size, 1), requires_grad=False)\n    causal_filters = th.zeros(data.shape[1], data.shape[1])\n\n    if gpu:\n        true_variable = true_variable.cuda(gpu_no)\n        false_variable = false_variable.cuda(gpu_no)\n        causal_filters = causal_filters.cuda(gpu_no)\n\n    data_iterator = DataLoader(data, batch_size=batch_size, shuffle=True)\n\n    # TRAIN\n    for epoch in range(train_epochs + test_epochs):\n        for i_batch, batch in enumerate(data_iterator):\n            batch = Variable(batch)\n            batch_vectors = [batch[:, [i]] for i in range(cols)]\n\n            g_optimizer.zero_grad()\n            d_optimizer.zero_grad()\n\n            # Train the discriminator\n            generated_variables = sam(batch)\n            disc_losses = []\n            gen_losses = []\n\n            for i in range(cols):\n                generator_output = th.cat([v for c in [batch_vectors[: i], [\n                    generated_variables[i]],\n                    batch_vectors[i + 1:]] for v in c], 1)\n                # 1. Train discriminator on fake\n                disc_output_detached = discriminator_sam(\n                    generator_output.detach())\n                disc_output = discriminator_sam(generator_output)\n                disc_losses.append(\n                    criterion(disc_output_detached, false_variable))\n\n                # 2. Train the generator :\n                gen_losses.append(criterion(disc_output, true_variable))\n\n            true_output = discriminator_sam(batch)\n            adv_loss = sum(disc_losses)/cols + \\\n                criterion(true_output, true_variable)\n            gen_loss = sum(gen_losses)\n\n            adv_loss.backward()\n            d_optimizer.step()\n\n            # 3. Compute filter regularization\n            filters = th.stack(\n                [i.fs_filter[0, :-1].abs() for i in sam.blocks], 1)\n            l1_reg = regul_param * filters.sum()\n            loss = gen_loss + l1_reg\n\n            if verbose and epoch % 200 == 0 and i_batch == 0:\n\n                print(str(i) + \" \" + d_str.format(epoch,\n                                                  adv_loss.item(),\n                                                  gen_loss.item() / cols,\n                                                  l1_reg.item()))\n            loss.backward()\n\n            # STORE ASSYMETRY values for output\n            if epoch > train_epochs:\n                causal_filters.add_(filters.data)\n            g_optimizer.step()\n\n            if plot:\n                plot_curves(i_batch, adv_loss, gen_loss, l1_reg, cols)\n\n            if plot_generated_pair and epoch % 200 == 0:\n                plot_gen(epoch, batch, generated_variables)\n\n    return causal_filters.div_(test_epochs).cpu().numpy()", "response": "Execute the SAM model."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef reset_parameters(self):\n        stdv = 1. / math.sqrt(self.weight.size(1))\n        self.weight.data.uniform_(-stdv, stdv)\n        if self.bias is not None:\n            self.bias.data.uniform_(-stdv, stdv)", "response": "Reset the parameters of the class."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef forward(self, input):\n        return th.nn.functional.linear(input, self.weight.div(self.weight.pow(2).sum(0).sqrt()))", "response": "Feed - forward through the network."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef forward(self, x):\n        return self.layers(x * (self._filter *\n                                self.fs_filter).expand_as(x))", "response": "Feed - forward the model."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nfeeding - forward the model.", "response": "def forward(self, x):\n        \"\"\"Feed-forward the model.\"\"\"\n        for i in self.noise:\n            i.data.normal_()\n\n        self.generated_variables = [self.blocks[i](\n            th.cat([x, self.noise[i]], 1)) for i in range(self.cols)]\n        return self.generated_variables"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nexecuting SAM on a dataset given a skeleton or not.", "response": "def predict(self, data, graph=None, nruns=6, njobs=None, gpus=0, verbose=None,\n                plot=False, plot_generated_pair=False, return_list_results=False):\n        \"\"\"Execute SAM on a dataset given a skeleton or not.\n\n        Args:\n            data (pandas.DataFrame): Observational data for estimation of causal relationships by SAM\n            skeleton (numpy.ndarray): A priori knowledge about the causal relationships as an adjacency matrix.\n                      Can be fed either directed or undirected links.\n            nruns (int): Number of runs to be made for causal estimation.\n                   Recommended: >=12 for optimal performance.\n            njobs (int): Numbers of jobs to be run in Parallel.\n                   Recommended: 1 if no GPU available, 2*number of GPUs else.\n            gpus (int): Number of available GPUs for the algorithm.\n            verbose (bool): verbose mode\n            plot (bool): Plot losses interactively. Not recommended if nruns>1\n            plot_generated_pair (bool): plots a generated pair interactively.  Not recommended if nruns>1\n        Returns:\n            networkx.DiGraph: Graph estimated by SAM, where A[i,j] is the term\n            of the ith variable for the jth generator.\n        \"\"\"\n        verbose, njobs = SETTINGS.get_default(('verbose', verbose), ('nb_jobs', njobs))\n        if njobs != 1:\n            list_out = Parallel(n_jobs=njobs)(delayed(run_SAM)(data,\n                                                               skeleton=graph,\n                                                               lr_gen=self.lr, lr_disc=self.dlr,\n                                                               regul_param=self.l1, nh=self.nh, dnh=self.dnh,\n                                                               gpu=bool(gpus), train_epochs=self.train,\n                                                               test_epochs=self.test, batch_size=self.batchsize,\n                                                               plot=plot, verbose=verbose, gpu_no=idx % max(gpus, 1))\n                                              for idx in range(nruns))\n        else:\n            list_out = [run_SAM(data, skeleton=graph,\n                                lr_gen=self.lr, lr_disc=self.dlr,\n                                regul_param=self.l1, nh=self.nh, dnh=self.dnh,\n                                gpu=bool(gpus), train_epochs=self.train,\n                                test_epochs=self.test, batch_size=self.batchsize,\n                                plot=plot, verbose=verbose, gpu_no=0)\n                        for idx in range(nruns)]\n        if return_list_results:\n            return list_out\n        else:\n            W = list_out[0]\n            for w in list_out[1:]:\n                W += w\n            W /= nruns\n        return nx.relabel_nodes(nx.DiGraph(W), {idx: i for idx, i in enumerate(data.columns)})"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ninfer causal relationships between two variables using the RECI statistic", "response": "def predict_proba(self, a, b, **kwargs):\n        \"\"\" Infer causal relationships between 2 variables using the RECI statistic\n\n        :param a: Input variable 1\n        :param b: Input variable 2\n        :return: Causation coefficient (Value : 1 if a->b and -1 if b->a)\n        :rtype: float\n        \"\"\"\n        return self.b_fit_score(b, a) - self.b_fit_score(a, b)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef b_fit_score(self, x, y):\n        x = np.reshape(minmax_scale(x), (-1, 1))\n        y = np.reshape(minmax_scale(y), (-1, 1))\n        poly = PolynomialFeatures(degree=self.degree)\n        poly_x = poly.fit_transform(x)\n\n        poly_x[:,1] = 0\n        poly_x[:,2] = 0\n\n        regressor = LinearRegression()\n        regressor.fit(poly_x, y)\n\n        y_predict = regressor.predict(poly_x)\n        error = mean_squared_error(y_predict, y)\n\n        return error", "response": "Compute the RECI fit score for a specific set of items."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef predict_proba(self, a, b, **kwargs):\n        return self.cds_score(b, a) - self.cds_score(a, b)", "response": "Infer causal relationships between two variables using the CDS statistic"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncompute the cds fit score from variable 1 to variable 2.", "response": "def cds_score(self, x_te, y_te):\n        \"\"\" Computes the cds statistic from variable 1 to variable 2\n\n        Args:\n            x_te (numpy.ndarray): Variable 1\n            y_te (numpy.ndarray): Variable 2\n\n        Returns:\n            float: CDS fit score\n        \"\"\"\n        if type(x_te) == np.ndarray:\n            x_te, y_te = pd.Series(x_te.reshape(-1)), pd.Series(y_te.reshape(-1))\n        xd, yd = discretized_sequences(x_te,  y_te,  self.ffactor, self.maxdev)\n        cx = Counter(xd)\n        cy = Counter(yd)\n        yrange = sorted(cy.keys())\n        ny = len(yrange)\n        py = np.array([cy[i] for i in yrange], dtype=float)\n        py = py / py.sum()\n        pyx = []\n        for a in cx:\n            if cx[a] > self.minc:\n                yx = y_te[xd == a]\n                # if not numerical(ty):\n                #     cyx = Counter(yx)\n                #     pyxa = np.array([cyx[i] for i in yrange], dtype=float)\n                #     pyxa.sort()\n                if count_unique(y_te) > len_discretized_values(y_te, \"Numerical\", self.ffactor, self.maxdev):\n\n                    yx = (yx - np.mean(yx)) / np.std(y_te)\n                    yx = discretized_sequence(yx, \"Numerical\", self.ffactor, self.maxdev, norm=False)\n                    cyx = Counter(yx.astype(int))\n                    pyxa = np.array([cyx[i] for i in discretized_values(y_te, \"Numerical\", self.ffactor, self.maxdev)],\n                                    dtype=float)\n\n                else:\n                    cyx = Counter(yx)\n                    pyxa = [cyx[i] for i in yrange]\n                    pyxax = np.array([0] * (ny - 1) + pyxa + [0] * (ny - 1), dtype=float)\n                    xcorr = [sum(py * pyxax[i:i + ny]) for i in range(2 * ny - 1)]\n                    imax = xcorr.index(max(xcorr))\n                    pyxa = np.array([0] * (2 * ny - 2 - imax) + pyxa + [0] * imax, dtype=float)\n                assert pyxa.sum() == cx[a]\n                pyxa = pyxa / pyxa.sum()\n\n                pyx.append(pyxa)\n\n        if len(pyx) == 0:\n            return 0\n\n        pyx = np.array(pyx)\n        pyx = pyx - pyx.mean(axis=0)\n        return np.std(pyx)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef predict_proba(self, a, b, **kwargs):\n        a = scale(a).reshape((-1, 1))\n        b = scale(b).reshape((-1, 1))\n\n        return self.anm_score(b, a) - self.anm_score(a, b)", "response": "Predicts the probability of a pair of causal income."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef anm_score(self, x, y):\n        gp = GaussianProcessRegressor().fit(x, y)\n        y_predict = gp.predict(x)\n        indepscore = normalized_hsic(y_predict - y, x)\n\n        return indepscore", "response": "Compute the fitness score of the ANM model in the x - direction y - direction"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef orient_undirected_graph(self, data, graph, **kwargs):\n        # Building setup w/ arguments.\n        self.arguments['{CITEST}'] = self.dir_CI_test[self.CI_test]\n        self.arguments['{METHOD_INDEP}'] = self.dir_method_indep[self.method_indep]\n        self.arguments['{DIRECTED}'] = 'TRUE'\n        self.arguments['{ALPHA}'] = str(self.alpha)\n        self.arguments['{NJOBS}'] = str(self.nb_jobs)\n        self.arguments['{VERBOSE}'] = str(self.verbose).upper()\n\n        fe = DataFrame(nx.adj_matrix(graph, weight=None).todense())\n        fg = DataFrame(1 - fe.values)\n\n        results = self._run_pc(data, fixedEdges=fe, fixedGaps=fg, verbose=self.verbose)\n\n        return nx.relabel_nodes(nx.DiGraph(results),\n                                {idx: i for idx, i in enumerate(data.columns)})", "response": "Run PC on an undirected graph."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nrunning the PC algorithm and return a networkx. DiGraph that represents the solution given by PC on the given data.", "response": "def create_graph_from_data(self, data, **kwargs):\n        \"\"\"Run the PC algorithm.\n\n        Args:\n            data (pandas.DataFrame): DataFrame containing the data\n\n        Returns:\n            networkx.DiGraph: Solution given by PC on the given data.\n       \"\"\"\n        # Building setup w/ arguments.\n        self.arguments['{CITEST}'] = self.dir_CI_test[self.CI_test]\n        self.arguments['{METHOD_INDEP}'] = self.dir_method_indep[self.method_indep]\n        self.arguments['{DIRECTED}'] = 'TRUE'\n        self.arguments['{ALPHA}'] = str(self.alpha)\n        self.arguments['{NJOBS}'] = str(self.nb_jobs)\n        self.arguments['{VERBOSE}'] = str(self.verbose).upper()\n\n        results = self._run_pc(data, verbose=self.verbose)\n\n        return nx.relabel_nodes(nx.DiGraph(results),\n                                {idx: i for idx, i in enumerate(data.columns)})"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _run_pc(self, data, fixedEdges=None, fixedGaps=None, verbose=True):\n        # Checking coherence of arguments\n        # print(self.arguments)\n        if (self.arguments['{CITEST}'] == self.dir_CI_test['hsic']\n           and self.arguments['{METHOD_INDEP}'] == self.dir_method_indep['corr']):\n            warnings.warn('Selected method for indep is unfit for the hsic test,'\n                          ' setting the hsic.gamma method.')\n            self.arguments['{METHOD_INDEP}'] = self.dir_method_indep['hsic_gamma']\n\n        elif (self.arguments['{CITEST}'] == self.dir_CI_test['gaussian']\n              and self.arguments['{METHOD_INDEP}'] != self.dir_method_indep['corr']):\n            warnings.warn('Selected method for indep is unfit for the selected test,'\n                          ' setting the classic correlation-based method.')\n            self.arguments['{METHOD_INDEP}'] = self.dir_method_indep['corr']\n\n        # Run PC\n        id = str(uuid.uuid4())\n        os.makedirs('/tmp/cdt_pc' + id + '/')\n        self.arguments['{FOLDER}'] = '/tmp/cdt_pc' + id + '/'\n\n        def retrieve_result():\n            return read_csv('/tmp/cdt_pc' + id + '/result.csv', delimiter=',').values\n\n        try:\n            data.to_csv('/tmp/cdt_pc' + id + '/data.csv', header=False, index=False)\n            if fixedGaps is not None and fixedEdges is not None:\n                fixedGaps.to_csv('/tmp/cdt_pc' + id + '/fixedgaps.csv', index=False, header=False)\n                fixedEdges.to_csv('/tmp/cdt_pc' + id + '/fixededges.csv', index=False, header=False)\n                self.arguments['{SKELETON}'] = 'TRUE'\n            else:\n                self.arguments['{SKELETON}'] = 'FALSE'\n\n            pc_result = launch_R_script(\"{}/R_templates/pc.R\".format(os.path.dirname(os.path.realpath(__file__))),\n                                        self.arguments, output_function=retrieve_result, verbose=verbose)\n        # Cleanup\n        except Exception as e:\n            rmtree('/tmp/cdt_pc' + id + '')\n            raise e\n        except KeyboardInterrupt:\n            rmtree('/tmp/cdt_pc' + id + '/')\n            raise KeyboardInterrupt\n        rmtree('/tmp/cdt_pc' + id + '')\n        return pc_result", "response": "Run the PC with all arguments."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef b_fit_score(self, x, y):\n        x = np.reshape(scale(x), (-1, 1))\n        y = np.reshape(scale(y), (-1, 1))\n        gp = GaussianProcessRegressor().fit(x, y)\n        y_predict = gp.predict(x)\n        error = mean_squared_error(y_predict, y)\n\n        return error", "response": "Computes the BF fit score for cds in the kernel."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef predict(self, data, alpha=0.01, max_iter=2000, **kwargs):\n        edge_model = GraphLasso(alpha=alpha, max_iter=max_iter)\n        edge_model.fit(data.values)\n\n        return nx.relabel_nodes(nx.DiGraph(edge_model.get_precision()),\n                                {idx: i for idx, i in enumerate(data.columns)})", "response": "Predict the graph skeleton."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef predict_features(self, df_features, df_target, idx=0, **kwargs):\n\n        y = np.transpose(df_target.values)\n        X = np.transpose(df_features.values)\n\n        path, beta, A, lam = hsiclasso(X, y)\n\n        return beta", "response": "For one variable predict its neighbouring nodes."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset the variable GPU and NB_JOBS to the number of CUDA devices.", "response": "def autoset_settings(set_var):\n    \"\"\"Autoset GPU parameters using CUDA_VISIBLE_DEVICES variables.\n\n    Return default config if variable not set.\n    :param set_var: Variable to set. Must be of type ConfigSettings\n    \"\"\"\n    try:\n        devices = ast.literal_eval(os.environ[\"CUDA_VISIBLE_DEVICES\"])\n        if type(devices) != list and type(devices) != tuple:\n            devices = [devices]\n        if len(devices) != 0:\n            set_var.GPU = len(devices)\n            set_var.NB_JOBS = len(devices)\n            warnings.warn(\"Detecting CUDA devices : {}\".format(devices))\n\n    except KeyError:\n        set_var.GPU = check_cuda_devices()\n        set_var.NB_JOBS = set_var.GPU\n        warnings.warn(\"Detecting {} CUDA devices.\".format(set_var.GPU))\n        if not set_var.GPU:\n            warnings.warn(\"No GPU automatically detected. Setting SETTINGS.GPU to 0, \" +\n                          \"and SETTINGS.NB_JOBS to cpu_count.\")\n            set_var.GPU = 0\n            set_var.NB_JOBS = multiprocessing.cpu_count()\n\n    return set_var"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef check_cuda_devices():\n    import ctypes\n\n    # Some constants taken from cuda.h\n    CUDA_SUCCESS = 0\n\n    libnames = ('libcuda.so', 'libcuda.dylib', 'cuda.dll')\n    for libname in libnames:\n        try:\n            cuda = ctypes.CDLL(libname)\n        except OSError:\n            continue\n        else:\n            break\n    else:\n        # raise OSError(\"could not load any of: \" + ' '.join(libnames))\n        return 0\n\n    nGpus = ctypes.c_int()\n    error_str = ctypes.c_char_p()\n\n    result = cuda.cuInit(0)\n    if result != CUDA_SUCCESS:\n        cuda.cuGetErrorString(result, ctypes.byref(error_str))\n        # print(\"cuInit failed with error code %d: %s\" % (result, error_str.value.decode()))\n        return 0\n    result = cuda.cuDeviceGetCount(ctypes.byref(nGpus))\n    if result != CUDA_SUCCESS:\n        cuda.cuGetErrorString(result, ctypes.byref(error_str))\n        # print(\"cuDeviceGetCount failed with error code %d: %s\" % (result, error_str.value.decode()))\n        return 0\n    # print(\"Found %d device(s).\" % nGpus.value)\n    return nGpus.value", "response": "Check CUDA devices on the computer and return the number of devices."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_default(self, *args, **kwargs):\n        def retrieve_param(i):\n            try:\n                return self.__getattribute__(i)\n            except AttributeError:\n                if i == \"device\":\n                    return self.default_device\n                else:\n                    return self.__getattribute__(i.upper())\n        if len(args) == 0:\n            if len(kwargs) == 1 and kwargs[list(kwargs.keys())[0]] is not None:\n                return kwargs[list(kwargs.keys())[0]]\n            elif len(kwargs) == 1:\n                return retrieve_param(list(kwargs.keys())[0])\n            else:\n                raise TypeError(\"As dict is unordered, it is impossible to give\"\n                                \"the parameters in the correct order.\")\n        else:\n            out = []\n            for i in args:\n                if i[1] is None:\n                    out.append(retrieve_param(i[0]))\n                else:\n                    out.append(i[1])\n            return out", "response": "This function returns the default value of the specified parameters."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef read_causal_pairs(filename, scale=True, **kwargs):\n    def convert_row(row, scale):\n        \"\"\"Convert a CCEPC row into numpy.ndarrays.\n\n        :param row:\n        :type row: pandas.Series\n        :return: tuple of sample ID and the converted data into numpy.ndarrays\n        :rtype: tuple\n        \"\"\"\n        a = row[\"A\"].split(\" \")\n        b = row[\"B\"].split(\" \")\n\n        if a[0] == \"\":\n            a.pop(0)\n            b.pop(0)\n        if a[-1] == \"\":\n            a.pop(-1)\n            b.pop(-1)\n\n        a = array([float(i) for i in a])\n        b = array([float(i) for i in b])\n        if scale:\n            a = scaler(a)\n            b = scaler(b)\n        return row['SampleID'], a, b\n    if isinstance(filename, str):\n        data = read_csv(filename, **kwargs)\n    elif isinstance(filename, DataFrame):\n        data = filename\n    else:\n        raise TypeError(\"Type not supported.\")\n    conv_data = []\n\n    for idx, row in data.iterrows():\n        conv_data.append(convert_row(row, scale))\n    df = DataFrame(conv_data, columns=['SampleID', 'A', 'B'])\n    df = df.set_index(\"SampleID\")\n    return df", "response": "Read a ChaLearn Cause effect pairs from a CSV file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreading a file containing an adjacency matrix and convert it into a networkx graph.", "response": "def read_adjacency_matrix(filename, directed=True, **kwargs):\n    \"\"\"Read a file (containing an adjacency matrix) and convert it into a\n    directed or undirected networkx graph.\n\n    :param filename: file to read or DataFrame containing the data\n    :type filename: str or pandas.DataFrame\n    :param directed: Return directed graph\n    :type directed: bool\n    :param kwargs: extra parameters to be passed to pandas.read_csv\n    :return: networkx graph containing the graph.\n    :rtype: **networkx.DiGraph** or **networkx.Graph** depending on the\n      ``directed`` parameter.\n    \"\"\"\n    if isinstance(filename, str):\n        data = read_csv(filename, **kwargs)\n    elif isinstance(filename, DataFrame):\n        data = filename\n    else:\n        raise TypeError(\"Type not supported.\")\n    if directed:\n        return nx.relabel_nodes(nx.DiGraph(data.values),\n                                {idx: i for idx, i in enumerate(data.columns)})\n    else:\n        return nx.relabel_nodes(nx.Graph(data.values),\n                                {idx: i for idx, i in enumerate(data.columns)})"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nread a file containing list of edges and return a directed networkx graph.", "response": "def read_list_edges(filename, directed=True, **kwargs):\n    \"\"\"Read a file (containing list of edges) and convert it into a directed\n    or undirected networkx graph.\n\n    :param filename: file to read or DataFrame containing the data\n    :type filename: str or pandas.DataFrame\n    :param directed: Return directed graph\n    :type directed: bool\n    :param kwargs: extra parameters to be passed to pandas.read_csv\n    :return: networkx graph containing the graph.\n    :rtype: **networkx.DiGraph** or **networkx.Graph** depending on the\n      ``directed`` parameter.\n\n    \"\"\"\n    if isinstance(filename, str):\n        data = read_csv(filename, **kwargs)\n    elif isinstance(filename, DataFrame):\n        data = filename\n    else:\n        raise TypeError(\"Type not supported.\")\n    if directed:\n        graph = nx.DiGraph()\n    else:\n        graph = nx.Graph()\n\n    for idx, row in data.iterrows():\n        try:\n            score = row[\"Score\"]\n        except KeyError:\n            score = 1\n        graph.add_edge(row['Cause'], row[\"Effect\"], weight=score)\n\n    return graph"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef forward(self, pred, target):\n        loss = th.FloatTensor([0])\n        for i in range(1, self.moments):\n            mk_pred = th.mean(th.pow(pred, i), 0)\n            mk_tar = th.mean(th.pow(target, i), 0)\n\n            loss.add_(th.mean((mk_pred - mk_tar) ** 2))  # L2\n\n        return loss", "response": "Compute the loss model."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef predict(self, a, b):\n        a = np.array(a).reshape((-1, 1))\n        b = np.array(b).reshape((-1, 1))\n        return (mutual_info_regression(a, b.reshape((-1,))) + mutual_info_regression(b, a.reshape((-1,))))/2", "response": "Compute the test statistic for the cluster cluster"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncomputes the test statistic for the statistic in the logarithmic system.", "response": "def predict(self, a, b):\n        \"\"\" Compute the test statistic\n\n        Args:\n            a (array-like): Variable 1\n            b (array-like): Variable 2\n\n        Returns:\n            float: test statistic\n        \"\"\"\n        a = np.array(a).reshape((-1, 1))\n        b = np.array(b).reshape((-1, 1))\n        return sp.kendalltau(a, b)[0]"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef predict(self, a, b, sig=[-1, -1], maxpnt=500):\n        a = (a - np.mean(a)) / np.std(a)\n        b = (b - np.mean(b)) / np.std(b)\n\n        return FastHsicTestGamma(a, b, sig, maxpnt)", "response": "Compute the test statistic for the current cluster and return it"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef predict(self, x, *args, **kwargs):\n        if len(args) > 0:\n            if type(args[0]) == nx.Graph or type(args[0]) == nx.DiGraph:\n                return self.orient_graph(x, *args, **kwargs)\n            else:\n                return self.predict_proba(x, *args, **kwargs)\n        elif type(x) == DataFrame:\n            return self.predict_dataset(x, *args, **kwargs)\n        elif type(x) == Series:\n            return self.predict_proba(x.iloc[0], x.iloc[1], *args, **kwargs)", "response": "Generic predict method. This method will execute the predict_proba method of the more specific subfunction."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef predict_dataset(self, x, **kwargs):\n        printout = kwargs.get(\"printout\", None)\n        pred = []\n        res = []\n        x.columns = [\"A\", \"B\"]\n        for idx, row in x.iterrows():\n            a = scale(row['A'].reshape((len(row['A']), 1)))\n            b = scale(row['B'].reshape((len(row['B']), 1)))\n\n            pred.append(self.predict_proba(a, b, idx=idx))\n\n            if printout is not None:\n                res.append([row['SampleID'], pred[-1]])\n                DataFrame(res, columns=['SampleID', 'Predictions']).to_csv(\n                    printout, index=False)\n        return pred", "response": "Generic dataset prediction function. Runs the score independently on all pairs of CEPC dataframes."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\norienting an undirected graph using the pairwise method defined by the subclass.", "response": "def orient_graph(self, df_data, graph, nb_runs=6, printout=None, **kwargs):\n        \"\"\"Orient an undirected graph using the pairwise method defined by the subclass.\n\n        The pairwise method is ran on every undirected edge.\n\n        Args:\n            df_data (pandas.DataFrame): Data\n            umg (networkx.Graph): Graph to orient\n            nb_runs (int): number of times to rerun for each pair (bootstrap)\n            printout (str): (optional) Path to file where to save temporary results\n\n        Returns:\n            networkx.DiGraph: a directed graph, which might contain cycles\n\n        .. warning:\n           Requirement : Name of the nodes in the graph correspond to name of\n           the variables in df_data\n        \"\"\"\n        if type(graph) == nx.DiGraph:\n            edges = [a for a in list(graph.edges()) if (a[1], a[0]) in list(graph.edges())]\n            oriented_edges = [a for a in list(graph.edges()) if (a[1], a[0]) not in list(graph.edges())]\n            for a in edges:\n                if (a[1], a[0]) in list(graph.edges()):\n                    edges.remove(a)\n            output = nx.DiGraph()\n            for i in oriented_edges:\n                output.add_edge(*i)\n\n        elif type(graph) == nx.Graph:\n            edges = list(graph.edges())\n            output = nx.DiGraph()\n\n        else:\n            raise TypeError(\"Data type not understood.\")\n\n        res = []\n\n        for idx, (a, b) in enumerate(edges):\n            weight = self.predict_proba(\n                df_data[a].values.reshape((-1, 1)), df_data[b].values.reshape((-1, 1)), idx=idx,\n                nb_runs=nb_runs, **kwargs)\n            if weight > 0:  # a causes b\n                output.add_edge(a, b, weight=weight)\n            else:\n                output.add_edge(b, a, weight=abs(weight))\n            if printout is not None:\n                res.append([str(a) + '-' + str(b), weight])\n                DataFrame(res, columns=['SampleID', 'Predictions']).to_csv(\n                    printout, index=False)\n\n        for node in list(df_data.columns.values):\n            if node not in output.nodes():\n                output.add_node(node)\n\n        return output"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nruns the algorithm on an undirected graph.", "response": "def orient_undirected_graph(self, data, graph):\n        \"\"\"Run the algorithm on an undirected graph.\n\n        Args:\n            data (pandas.DataFrame): DataFrame containing the data\n            graph (networkx.Graph): Skeleton of the graph to orient\n\n        Returns:\n            networkx.DiGraph: Solution on the given skeleton.\n\n        \"\"\"\n        # Building setup w/ arguments.\n        self.arguments['{VERBOSE}'] = str(self.verbose).upper()\n        self.arguments['{SCORE}'] = self.score\n        self.arguments['{BETA}'] = str(self.beta)\n        self.arguments['{OPTIM}'] = str(self.optim).upper()\n        self.arguments['{ALPHA}'] = str(self.alpha)\n\n        whitelist = DataFrame(list(nx.edges(graph)), columns=[\"from\", \"to\"])\n        blacklist = DataFrame(list(nx.edges(nx.DiGraph(DataFrame(-nx.adj_matrix(graph, weight=None).to_dense() + 1,\n                                                                 columns=list(graph.nodes()),\n                                                                 index=list(graph.nodes()))))), columns=[\"from\", \"to\"])\n        results = self._run_bnlearn(data, whitelist=whitelist,\n                                   blacklist=blacklist, verbose=self.verbose)\n\n        return nx.relabel_nodes(nx.DiGraph(results),\n                                {idx: i for idx, i in enumerate(data.columns)})"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef orient_directed_graph(self, data, graph):\n        warnings.warn(\"The algorithm is ran on the skeleton of the given graph.\")\n        return self.orient_undirected_graph(data, nx.Graph(graph))", "response": "Run the algorithm on a directed_graph."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun the algorithm on data and return a networkx. DiGraph", "response": "def create_graph_from_data(self, data):\n        \"\"\"Run the algorithm on data.\n\n        Args:\n            data (pandas.DataFrame): DataFrame containing the data\n\n        Returns:\n            networkx.DiGraph: Solution given by the algorithm.\n\n        \"\"\"\n        # Building setup w/ arguments.\n        self.arguments['{SCORE}'] = self.score\n        self.arguments['{VERBOSE}'] = str(self.verbose).upper()\n        self.arguments['{BETA}'] = str(self.beta)\n        self.arguments['{OPTIM}'] = str(self.optim).upper()\n        self.arguments['{ALPHA}'] = str(self.alpha)\n\n        results = self._run_bnlearn(data, verbose=self.verbose)\n        graph = nx.DiGraph()\n        graph.add_edges_from(results)\n        return graph"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncomputing the gaussian kernel on a 1D vector.", "response": "def computeGaussKernel(x):\n    \"\"\"Compute the gaussian kernel on a 1D vector.\"\"\"\n    xnorm = np.power(euclidean_distances(x, x), 2)\n    return np.exp(-xnorm / (2.0))"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef normal_noise(points):\n    return np.random.rand(1) * np.random.randn(points, 1) \\\n        + random.sample([2, -2], 1)", "response": "Init a noise variable."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef predict_dataset(self, df):\n        if len(list(df.columns)) == 2:\n            df.columns = [\"A\", \"B\"]\n        if self.model is None:\n            raise AssertionError(\"Model has not been trained before predictions\")\n        df2 = DataFrame()\n\n        for idx, row in df.iterrows():\n            df2 = df2.append(row, ignore_index=True)\n            df2 = df2.append({'A': row[\"B\"], 'B': row[\"A\"]}, ignore_index=True)\n        return predict.predict(deepcopy(df2), deepcopy(self.model))[::2]", "response": "Runs Jarfo independently on all pairs of CEPC dataframes."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nusing Jarfo to predict the causal direction of a pair of vars.", "response": "def predict_proba(self, a, b, idx=0, **kwargs):\n        \"\"\" Use Jarfo to predict the causal direction of a pair of vars.\n\n        Args:\n            a (numpy.ndarray): Variable 1\n            b (numpy.ndarray): Variable 2\n            idx (int): (optional) index number for printing purposes\n\n        Returns:\n            float: Causation score (Value : 1 if a->b and -1 if b->a)\n        \"\"\"\n        return self.predict_dataset(DataFrame([[a, b]],\n                                              columns=['A', 'B']))"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef network_deconvolution(mat, **kwargs):\n    alpha = kwargs.get('alpha', 1)\n    beta = kwargs.get('beta', 0.99)\n    control = kwargs.get('control', 0)\n\n    # ToDO : ASSERTS\n    try:\n        assert beta < 1 or beta > 0\n        assert alpha <= 1 or alpha > 0\n\n    except AssertionError:\n        raise ValueError(\"alpha must be in ]0, 1] and beta in [0, 1]\")\n\n    #  Processing the input matrix, diagonal values are filtered\n    np.fill_diagonal(mat, 0)\n\n    # Thresholding the input matrix\n    y = stat.mquantiles(mat[:], prob=[1 - alpha])\n    th = mat >= y\n    mat_th = mat * th\n\n    # Making the matrix symetric if already not\n    mat_th = (mat_th + mat_th.T) / 2\n\n    # Eigen decomposition\n    Dv, U = LA.eigh(mat_th)\n    D = np.diag((Dv))\n    lam_n = np.abs(np.min(np.min(np.diag(D)), 0))\n    lam_p = np.abs(np.max(np.max(np.diag(D)), 0))\n\n    m1 = lam_p * (1 - beta) / beta\n    m2 = lam_n * (1 + beta) / beta\n    m = max(m1, m2)\n\n    # network deconvolution\n    for i in range(D.shape[0]):\n        D[i, i] = (D[i, i]) / (m + D[i, i])\n\n    mat_new1 = np.dot(U, np.dot(D, LA.inv(U)))\n\n    # Displying direct weights\n\n    if control == 0:\n        ind_edges = (mat_th > 0) * 1.0\n        ind_nonedges = (mat_th == 0) * 1.0\n        m1 = np.max(np.max(mat * ind_nonedges))\n        m2 = np.min(np.min(mat_new1))\n        mat_new2 = (mat_new1 + np.max(m1 - m2, 0)) * ind_edges + (mat * ind_nonedges)\n    else:\n        m2 = np.min(np.min(mat_new1))\n        mat_new2 = (mat_new1 + np.max(-m2, 0))\n\n    # linearly mapping the deconvolved matrix to be between 0 and 1\n    m1 = np.min(np.min(mat_new2))\n    m2 = np.max(np.max(mat_new2))\n    mat_nd = (mat_new2 - m1) / (m2 - m1)\n\n    return mat_nd", "response": "Python implementation of network deconvolution by MIT - KELLIS LAB."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nremove undirected links from a graph.", "response": "def remove_indirect_links(g, alg=\"aracne\", **kwargs):\n    \"\"\"Apply deconvolution to a networkx graph.\n\n    Args:\n       g (networkx.Graph): Graph to apply deconvolution to\n       alg (str): Algorithm to use ('aracne', 'clr', 'nd')\n       kwargs (dict): extra options for algorithms\n\n    Returns:\n       networkx.Graph: graph with undirected links removed.\n    \"\"\"\n    alg = {\"aracne\": aracne,\n           \"nd\": network_deconvolution,\n           \"clr\": clr}[alg]\n    mat = np.array(nx.adjacency_matrix(g).todense())\n    return nx.relabel_nodes(nx.DiGraph(alg(mat, **kwargs)),\n                            {idx: i for idx, i in enumerate(list(g.nodes()))})"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef dagify_min_edge(g):\n    while not nx.is_directed_acyclic_graph(g):\n        cycle = next(nx.simple_cycles(g))\n        scores = []\n        edges = []\n        for i, j in zip(cycle[:1], cycle[:1]):\n            edges.append((i, j))\n            scores.append(g[i][j]['weight'])\n\n        i, j = edges[scores.index(min(scores))]\n        gc = deepcopy(g)\n        gc.remove_edge(i, j)\n        gc.add_edge(j, i)\n\n        if len(list(nx.simple_cycles(gc))) < len(list(nx.simple_cycles(g))):\n            g.add_edge(j, i, weight=min(scores))\n        g.remove_edge(i, j)\n    return g", "response": "Input a graph and output a DAG with the lowest score of the cycle\n   ."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns the weighted average and standard deviation.", "response": "def weighted_mean_and_std(values, weights):\n    \"\"\"\n    Returns the weighted average and standard deviation.\n\n    values, weights -- numpy ndarrays with the same shape.\n    \"\"\"\n    average = np.average(values, weights=weights, axis=0)\n    variance = np.dot(weights, (values - average) ** 2) / weights.sum()  # Fast and numerically precise\n    return (average, np.sqrt(variance))"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nrun an instance of GNN on a GPU.", "response": "def GNN_instance(x, idx=0, device=None, nh=20, **kwargs):\n    \"\"\"Run an instance of GNN, testing causal direction.\n\n    :param m: data corresponding to the config : (N, 2) data, [:, 0] cause and [:, 1] effect\n    :param pair_idx: print purposes\n    :param run: numner of the run (for GPU dispatch)\n    :param device: device on with the algorithm is going to be run on.\n    :return:\n    \"\"\"\n    device = SETTINGS.get_default(device=device)\n    xy = scale(x).astype('float32')\n    inputx = th.FloatTensor(xy[:, [0]]).to(device)\n    target = th.FloatTensor(xy[:, [1]]).to(device)\n    GNNXY = GNN_model(x.shape[0], device=device, nh=nh).to(device)\n    GNNYX = GNN_model(x.shape[0], device=device, nh=nh).to(device)\n    GNNXY.reset_parameters()\n    GNNYX.reset_parameters()\n    XY = GNNXY.run(inputx, target, **kwargs)\n    YX = GNNYX.run(target, inputx, **kwargs)\n\n    return [XY, YX]"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\npass data through the net structure.", "response": "def forward(self, x):\n        \"\"\"Pass data through the net structure.\n\n        :param x: input data: shape (:,1)\n        :type x: torch.Variable\n        :return: output of the shallow net\n        :rtype: torch.Variable\n\n        \"\"\"\n        self.noise.normal_()\n        return self.layers(th.cat([x, self.noise], 1))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef run(self, x, y, lr=0.01, train_epochs=1000, test_epochs=1000, idx=0, verbose=None, **kwargs):\n        verbose = SETTINGS.get_default(verbose=verbose)\n        optim = th.optim.Adam(self.parameters(), lr=lr)\n        running_loss = 0\n        teloss = 0\n\n        for i in range(train_epochs + test_epochs):\n            optim.zero_grad()\n            pred = self.forward(x)\n            loss = self.criterion(pred, y)\n            running_loss += loss.item()\n\n            if i < train_epochs:\n                loss.backward()\n                optim.step()\n            else:\n                teloss += running_loss\n\n            # print statistics\n            if verbose and not i % 300:\n                print('Idx:{}; epoch:{}; score:{}'.\n                      format(idx, i, running_loss/300))\n                running_loss = 0.0\n\n        return teloss / test_epochs", "response": "Run the GNN on a pair x y."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nruns multiple times GNN to estimate the causal direction of the pair.", "response": "def predict_proba(self, a, b, nb_runs=6, nb_jobs=None, gpu=None,\n                      idx=0, verbose=None, ttest_threshold=0.01,\n                      nb_max_runs=16, train_epochs=1000, test_epochs=1000):\n        \"\"\"Run multiple times GNN to estimate the causal direction.\n\n        Args:\n            a (np.ndarray): Variable 1\n            b (np.ndarray): Variable 2\n            nb_runs (int): number of runs to execute per batch (before testing for significance with t-test).\n            nb_jobs (int): number of runs to execute in parallel. (Initialized with ``cdt.SETTINGS.NB_JOBS``)\n            gpu (bool): use gpu (Initialized with ``cdt.SETTINGS.GPU``)\n            idx (int): (optional) index of the pair, for printing purposes\n            verbose (bool): verbosity (Initialized with ``cdt.SETTINGS.verbose``)\n            ttest_threshold (float): threshold to stop the boostraps before ``nb_max_runs`` if the difference is significant\n            nb_max_runs (int): Max number of bootstraps\n            train_epochs (int): Number of epochs during which the model is going to be trained\n            test_epochs (int): Number of epochs during which the model is going to be tested\n\n        Returns:\n            float: Causal score of the pair (Value : 1 if a->b and -1 if b->a)\n        \"\"\"\n        Nb_jobs, verbose, gpu = SETTINGS.get_default(('nb_jobs', nb_jobs), ('verbose', verbose), ('gpu', gpu))\n        x = np.stack([a.ravel(), b.ravel()], 1)\n        ttest_criterion = TTestCriterion(\n            max_iter=nb_max_runs, runs_per_iter=nb_runs, threshold=ttest_threshold)\n\n        AB = []\n        BA = []\n\n        while ttest_criterion.loop(AB, BA):\n            if nb_jobs != 1:\n                result_pair = Parallel(n_jobs=nb_jobs)(delayed(GNN_instance)(\n                    x, idx=idx, device='cuda:{}'.format(run % gpu) if gpu else 'cpu',\n                    verbose=verbose, train_epochs=train_epochs, test_epochs=test_epochs) for run in range(ttest_criterion.iter, ttest_criterion.iter + nb_runs))\n            else:\n                result_pair = [GNN_instance(x, idx=idx,\n                                            device='cuda:0' if gpu else 'cpu',\n                                            verbose=verbose,\n                                            train_epochs=train_epochs,\n                                            test_epochs=test_epochs)\n                               for run in range(ttest_criterion.iter, ttest_criterion.iter + nb_runs)]\n            AB.extend([runpair[0] for runpair in result_pair])\n            BA.extend([runpair[1] for runpair in result_pair])\n\n        if verbose:\n            print(\"P-value after {} runs : {}\".format(ttest_criterion.iter,\n                                                      ttest_criterion.p_value))\n\n        score_AB = np.mean(AB)\n        score_BA = np.mean(BA)\n\n        return (score_BA - score_AB) / (score_BA + score_AB)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef init_variables(self, verbose=False):\n        # Resetting adjacency matrix\n        for i in range(self.nodes):\n            for j in np.random.choice(range(self.nodes),\n                                      np.random.randint(\n                                          0, self.parents_max + 1),\n                                      replace=False):\n                if i != j:\n                    self.adjacency_matrix[j, i] = 1\n\n        try:\n            assert any([sum(self.adjacency_matrix[:, i]) ==\n                        self.parents_max for i in range(self.nodes)])\n            self.g = nx.DiGraph(self.adjacency_matrix)\n            assert list(nx.simple_cycles(self.g))\n            assert any(len(i) == 2 for i in nx.simple_cycles(self.g))\n\n        except AssertionError:\n            if verbose:\n                print(\"Regenerating, graph non valid...\")\n            self.init_variables()\n\n        if verbose:\n            print(\"Matrix generated ! \\\n              Number of cycles: {}\".format(len(list(nx.simple_cycles(self.g)))))\n\n        for i in range(self.nodes):\n            self.data.iloc[:, i] = scale(self.initial_generator(self.points))\n\n        # Mechanisms\n        self.cfunctions = [self.mechanism(int(sum(self.adjacency_matrix[:, i])),\n                                          self.points, self.noise, noise_coeff=self.noise_coeff) for i in range(self.nodes)]", "response": "Redefine the causes of the graph."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef generate(self, nb_steps=100, averaging=50, rescale=True):\n        if self.cfunctions is None:\n            self.init_variables()\n        new_df = pd.DataFrame()\n        causes = [[c for c in np.nonzero(self.adjacency_matrix[:, j])[0]]\n                  for j in range(self.nodes)]\n        values = [[] for i in range(self.nodes)]\n\n        for i in range(nb_steps):\n            for j in range(self.nodes):\n                new_df[\"V\" + str(j)] = self.cfunctions[j](self.data.iloc[:, causes[j]].values)[:, 0]\n                if rescale:\n                    new_df[\"V\" + str(j)] = scale(new_df[\"V\" + str(j)])\n                if i > nb_steps-averaging:\n                    values[j].append(new_df[\"V\" + str(j)])\n            self.data = new_df\n        self.data = pd.DataFrame(np.array([np.mean(values[i], axis=0)\n                                           for i in range(self.nodes)]).transpose(),\n                                 columns=[\"V{}\".format(j) for j in range(self.nodes)])\n\n        return self.g, self.data", "response": "Generate data from an FCM containing cycles."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_graph_from_data(self, data, **kwargs):\n        # Building setup w/ arguments.\n        self.arguments['{SCORE}'] = self.scores[self.score]\n        self.arguments['{CUTOFF}'] = str(self.cutoff)\n        self.arguments['{VARSEL}'] = str(self.variablesel).upper()\n        self.arguments['{SELMETHOD}'] = self.var_selection[self.selmethod]\n        self.arguments['{PRUNING}'] = str(self.pruning).upper()\n        self.arguments['{PRUNMETHOD}'] = self.var_selection[self.prunmethod]\n        self.arguments['{NJOBS}'] = str(self.nb_jobs)\n        self.arguments['{VERBOSE}'] = str(self.verbose).upper()\n        results = self._run_cam(data, verbose=self.verbose)\n\n        return nx.relabel_nodes(nx.DiGraph(results),\n                                {idx: i for idx, i in enumerate(data.columns)})", "response": "This method creates a networkx. DiGraph from the data provided by the CAM algorithm."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef predict_features(self, df_features, df_target, idx=0, **kwargs):\n        estimator = SVR(kernel='linear')\n        selector = RFECV(estimator, step=1)\n        selector = selector.fit(df_features.values, df_target.values[:, 0])\n\n        return selector.grid_scores_", "response": "For one variable predict its neighbouring nodes."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef forward(self, x):\n\n        features = self.conv(x).mean(dim=2)\n        return self.dense(features)", "response": "Passing data through the network."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nfit the NCC model to the given data.", "response": "def fit(self, x_tr, y_tr, epochs=50, batchsize=32,\n            learning_rate=0.01, verbose=None, device=None):\n        \"\"\"Fit the NCC model.\n\n        Args:\n            x_tr (pd.DataFrame): CEPC format dataframe containing the pairs\n            y_tr (pd.DataFrame or np.ndarray): labels associated to the pairs\n            epochs (int): number of train epochs\n            learning_rate (float): learning rate of Adam\n            verbose (bool): verbosity (defaults to ``cdt.SETTINGS.verbose``)\n            device (str): cuda or cpu device (defaults to ``cdt.SETTINGS.default_device``)\n        \"\"\"\n        if batchsize > len(x_tr):\n            batchsize = len(x_tr)\n        verbose, device = SETTINGS.get_default(('verbose', verbose),\n                                               ('device', device))\n        self.model = NCC_model()\n        opt = th.optim.Adam(self.model.parameters(), lr=learning_rate)\n        criterion = th.nn.BCEWithLogitsLoss()\n        y = y_tr.values if isinstance(y_tr, pd.DataFrame) else y_tr\n        y = th.Tensor(y)/2 + .5\n        # print(y)\n        self.model = self.model.to(device)\n        y = y.to(device)\n        dataset = []\n        for i, (idx, row) in enumerate(x_tr.iterrows()):\n\n            a = row['A'].reshape((len(row['A']), 1))\n            b = row['B'].reshape((len(row['B']), 1))\n            m = np.hstack((a, b))\n            m = m.astype('float32')\n            m = th.from_numpy(m).t().unsqueeze(0)\n            dataset.append(m)\n        dataset = [m.to(device) for m in dataset]\n        acc = [0]\n        da = th.utils.data.DataLoader(Dataset(dataset, y), batch_size=batchsize,\n                                      shuffle=True)\n        data_per_epoch = (len(dataset) // batchsize)\n        with trange(epochs, desc=\"Epochs\", disable=not verbose) as te:\n            for epoch in te:\n                with trange(data_per_epoch, desc=\"Batches of {}\".format(batchsize),\n                            disable=not (verbose and batchsize == len(dataset))) as t:\n                    output = []\n                    labels = []\n                    for (batch, label), i in zip(da, t):\n                        opt.zero_grad()\n                        # print(batch.shape, labels.shape)\n                        out = th.stack([self.model(m) for m in batch], 0).squeeze(2)\n                        loss = criterion(out, label)\n                        loss.backward()\n                        t.set_postfix(loss=loss.item())\n                        opt.step()\n                        output.append(out)\n                        labels.append(label)\n                    acc = th.where(th.cat(output, 0) > .5,\n                                   th.ones(len(output)),\n                                   th.zeros(len(output))) - th.cat(labels, 0)\n                    te.set_postfix(Acc=1-acc.abs().mean().item())"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\npredicts causal directions using the trained NCC pairwise model.", "response": "def predict_proba(self, a, b, device=None):\n        \"\"\"Infer causal directions using the trained NCC pairwise model.\n\n        Args:\n            a (numpy.ndarray): Variable 1\n            b (numpy.ndarray): Variable 2\n            device (str): Device to run the algorithm on (defaults to ``cdt.SETTINGS.default_device``)\n\n        Returns:\n            float: Causation score (Value : 1 if a->b and -1 if b->a)\n        \"\"\"\n        device = SETTINGS.get_default(device=device)\n        if self.model is None:\n            print('Model has to be trained before doing any predictions')\n            raise ValueError\n        if len(np.array(a).shape) == 1:\n            a = np.array(a).reshape((-1, 1))\n            b = np.array(b).reshape((-1, 1))\n        m = np.hstack((a, b))\n        m = scale(m)\n        m = m.astype('float32')\n        m = th.from_numpy(m).t().unsqueeze(0)\n\n        if th.cuda.is_available():\n            m = m.cuda()\n\n        return (self.model(m).data.cpu().numpy()-.5) * 2"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef predict_dataset(self, df, device=None, verbose=None):\n        verbose, device = SETTINGS.get_default(('verbose', verbose),\n                                               ('device', device))\n        dataset = []\n        for i, (idx, row) in enumerate(df.iterrows()):\n            a = row['A'].reshape((len(row['A']), 1))\n            b = row['B'].reshape((len(row['B']), 1))\n            m = np.hstack((a, b))\n            m = m.astype('float32')\n            m = th.from_numpy(m).t().unsqueeze(0)\n            dataset.append(m)\n\n        dataset = [m.to(device) for m in dataset]\n        return pd.DataFrame((th.cat([self.model(m) for m, t in zip(dataset, trange(len(dataset)),\n                                                      disable=not verbose)]\\\n                       , 0).data.cpu().numpy() -.5) * 2)", "response": "Predicts the causation coefficients for the given pairs of causal classes."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef phrase_to_filename(self, phrase):\n        # remove non-word characters\n        name = re.sub(r\"[^\\w\\s\\.]\", '', phrase.strip().lower())\n        # replace whitespace with underscores\n        name = re.sub(r\"\\s+\", '_', name)\n\n        return name + '.png'", "response": "Convert a phrase to a normilized file name."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef open(self):\n        if self.seed_url:\n            self.driver_adapter.open(self.seed_url)\n            self.wait_for_page_to_load()\n            return self\n        raise UsageError(\"Set a base URL or URL_TEMPLATE to open this page.\")", "response": "Open the page.\n\n        Navigates to :py:attr:`seed_url` and calls :py:func:`wait_for_page_to_load`.\n\n        :return: The current page object.\n        :rtype: :py:class:`Page`\n        :raises: UsageError"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef wait_for_page_to_load(self):\n        self.wait.until(lambda _: self.loaded)\n        self.pm.hook.pypom_after_wait_for_page_to_load(page=self)\n        return self", "response": "Wait for the page to load."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef register():\n    registerDriver(\n        ISelenium,\n        Selenium,\n        class_implements=[\n            Firefox,\n            Chrome,\n            Ie,\n            Edge,\n            Opera,\n            Safari,\n            BlackBerry,\n            PhantomJS,\n            Android,\n            Remote,\n            EventFiringWebDriver,\n        ],\n    )", "response": "Register the Selenium specific driver implementation."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef root(self):\n        if self._root is None and self._root_locator is not None:\n            return self.page.find_element(*self._root_locator)\n        return self._root", "response": "Return the root element for the page region."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef wait_for_region_to_load(self):\n        self.wait.until(lambda _: self.loaded)\n        self.pm.hook.pypom_after_wait_for_region_to_load(region=self)\n        return self", "response": "Wait for the page region to load."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef find_element(self, strategy, locator):\n        return self.driver_adapter.find_element(strategy, locator, root=self.root)", "response": "Finds an element on the page."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef find_elements(self, strategy, locator):\n        return self.driver_adapter.find_elements(strategy, locator, root=self.root)", "response": "Finds elements on the page."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nchecking whether an element is present.", "response": "def is_element_present(self, strategy, locator):\n        \"\"\"Checks whether an element is present.\n\n        :param strategy: Location strategy to use. See :py:class:`~selenium.webdriver.common.by.By` or :py:attr:`~pypom.splinter_driver.ALLOWED_STRATEGIES`.\n        :param locator: Location of target element.\n        :type strategy: str\n        :type locator: str\n        :return: ``True`` if element is present, else ``False``.\n        :rtype: bool\n\n        \"\"\"\n        return self.driver_adapter.is_element_present(strategy, locator, root=self.root)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nchecks whether an element is displayed.", "response": "def is_element_displayed(self, strategy, locator):\n        \"\"\"Checks whether an element is displayed.\n\n        :param strategy: Location strategy to use. See :py:class:`~selenium.webdriver.common.by.By` or :py:attr:`~pypom.splinter_driver.ALLOWED_STRATEGIES`.\n        :param locator: Location of target element.\n        :type strategy: str\n        :type locator: str\n        :return: ``True`` if element is displayed, else ``False``.\n        :rtype: bool\n\n        \"\"\"\n        return self.driver_adapter.is_element_displayed(\n            strategy, locator, root=self.root\n        )"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nregister driver adapter used by page object", "response": "def registerDriver(iface, driver, class_implements=[]):\n    \"\"\" Register driver adapter used by page object\"\"\"\n    for class_item in class_implements:\n        classImplements(class_item, iface)\n\n    component.provideAdapter(factory=driver, adapts=[iface], provides=IDriver)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef isHcl(s):\n    '''\n        Detects whether a string is JSON or HCL\n        \n        :param s: String that may contain HCL or JSON\n        \n        :returns: True if HCL, False if JSON, raises ValueError\n                  if neither\n    '''\n    for c in s:\n        if c.isspace():\n            continue\n\n        if c == '{':\n            return False\n        else:\n            return True\n\n    raise ValueError(\"No HCL object could be decoded\")", "response": "Detects whether a string contains HCL or JSON or HCL."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef t_string(self, t):\n        # Start of a string\n        r'\\\"'\n        # abs_start is the absolute start of the string. We use this at the end\n        # to know how many new lines we've consumed\n        t.lexer.abs_start = t.lexer.lexpos\n        # rel_pos is the begining of the unconsumed part of the string. It will\n        # get modified when consuming escaped characters\n        t.lexer.rel_pos = t.lexer.lexpos\n        # The value of the consumed part of the string\n        t.lexer.string_value = u''\n        t.lexer.begin('string')", "response": "Tokenizes the string of the log entries."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nhandles the escaped character in the string.", "response": "def t_string_escapedchar(self, t):\n        # If a quote or backslash is escaped, build up the string by ignoring\n        # the escape character. Should this be done for other characters?\n        r'(?<=\\\\)(\\\"|\\\\)'\n        t.lexer.string_value += (\n            t.lexer.lexdata[t.lexer.rel_pos : t.lexer.lexpos - 2] + t.value\n        )\n        t.lexer.rel_pos = t.lexer.lexpos\n        pass"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef t_stringdollar_rbrace(self, t):\n        r'\\}'\n        t.lexer.braces -= 1\n\n        if t.lexer.braces == 0:\n            # End of the dollar brace, back to the rest of the string\n            t.lexer.begin('string')", "response": "Tokenize a dollar brace."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef t_tabbedheredoc(self, t):\n        r'<<-\\S+\\r?\\n'\n        t.lexer.is_tabbed = True\n        self._init_heredoc(t)\n        t.lexer.begin('tabbedheredoc')", "response": "r << - S+\\ r?\\n"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef objectlist_flat(self, lt, replace):\n        '''\n            Similar to the dict constructor, but handles dups\n            \n            HCL is unclear on what one should do when duplicate keys are\n            encountered. These comments aren't clear either:\n            \n            from decoder.go: if we're at the root or we're directly within\n                             a list, decode into dicts, otherwise lists\n                \n            from object.go: there's a flattened list structure\n        '''\n        d = {}\n\n        for k, v in lt:\n            if k in d.keys() and not replace:\n                if type(d[k]) is list:\n                    d[k].append(v)\n                else:\n                    d[k] = [d[k], v]\n            else:\n                if isinstance(v, dict):\n                    dd = d.setdefault(k, {})\n                    for kk, vv in iteritems(v):\n                        if type(dd) == list:\n                            dd.append({kk: vv})\n                        elif kk in dd.keys():\n                            if hasattr(vv, 'items'):\n                                for k2, v2 in iteritems(vv):\n                                    dd[kk][k2] = v2\n                            else:\n                                d[k] = [dd, {kk: vv}]\n                        else:\n                            dd[kk] = vv\n                else:\n                    d[k] = v\n\n        return d", "response": "This function takes a list of objects and returns a flattened list of objects."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef p_objectlist_2(self, p):\n        \"objectlist : objectlist COMMA objectitem\"\n        if DEBUG:\n            self.print_p(p)\n        p[0] = p[1] + [p[3]]", "response": "objectlist : objectlist COMMA objectitem"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nobject : LEFTBRACE objectlist RIGHTBRACE", "response": "def p_object_0(self, p):\n        \"object : LEFTBRACE objectlist RIGHTBRACE\"\n        if DEBUG:\n            self.print_p(p)\n        p[0] = self.objectlist_flat(p[2], False)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nobjecting : LEFTBRACE objectlist COMMA RIGHTBRACE", "response": "def p_object_1(self, p):\n        \"object : LEFTBRACE objectlist COMMA RIGHTBRACE\"\n        if DEBUG:\n            self.print_p(p)\n        p[0] = self.objectlist_flat(p[2], False)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef p_objectitem_0(self, p):\n        '''\n        objectitem : objectkey EQUAL number\n                   | objectkey EQUAL BOOL\n                   | objectkey EQUAL STRING\n                   | objectkey EQUAL object\n                   | objectkey EQUAL list\n        '''\n        if DEBUG:\n            self.print_p(p)\n        p[0] = (p[1], p[3])", "response": "objectitem : objectkey EQUAL number\n                   | objectkey EQUAL BOOL\n                   | objectkey EQUAL STRING\n                   | objectkey EQUAL object\n                   | objectkey EQUAL list"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nblocking : blockId object", "response": "def p_block_0(self, p):\n        \"block : blockId object\"\n        if DEBUG:\n            self.print_p(p)\n        p[0] = (p[1], p[2])"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef p_block_1(self, p):\n        \"block : blockId block\"\n        if DEBUG:\n            self.print_p(p)\n        p[0] = (p[1], {p[2][0]: p[2][1]})", "response": "block : blockId block"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef p_listitems_1(self, p):\n        \"listitems : listitems COMMA listitem\"\n        if DEBUG:\n            self.print_p(p)\n        p[0] = p[1] + [p[3]]", "response": "listitems : listitems COMMA listitem"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nnumber : int exp", "response": "def p_number_2(self, p):\n        \"number : int exp\"\n        if DEBUG:\n            self.print_p(p)\n        p[0] = float(\"{0}{1}\".format(p[1], p[2]))"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef p_number_3(self, p):\n        \"number : float exp\"\n        if DEBUG:\n            self.print_p(p)\n        p[0] = float(\"{0}{1}\".format(p[1], p[2]))", "response": "number : float exp"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ninitializes the parse table at install time", "response": "def _pre_install():\n    '''Initialize the parse table at install time'''\n\n    # Generate the parsetab.dat file at setup time\n    dat = join(setup_dir, 'src', 'hcl', 'parsetab.dat')\n    if exists(dat):\n        os.unlink(dat)\n\n    sys.path.insert(0, join(setup_dir, 'src'))\n\n    import hcl\n    from hcl.parser import HclParser\n\n    parser = HclParser()"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef append(self, linenumber, raw_text, cells):\n        self.rows.append(Row(linenumber, raw_text, cells))", "response": "Add a new row of data from a test suite"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list of statements that are not settings or comments.", "response": "def steps(self):\n        \"\"\"Return a list of steps (statements that are not settings or comments)\"\"\"\n        steps = []\n        for statement in self.statements:\n            if ((not statement.is_comment()) and\n                (not statement.is_setting())):\n                steps.append(statement)\n        return steps"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef is_comment(self):\n        '''Return True if the first non-empty cell starts with \"#\"'''\n\n        for cell in self[:]:\n            if cell == \"\":\n                continue\n\n            # this is the first non-empty cell. Check whether it is\n            # a comment or not.\n            if cell.lstrip().startswith(\"#\"):\n                return True\n            else:\n                return False\n        return False", "response": "Return True if the first non - empty cell starts with \"#\"'"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning an instance of SuiteFile ResourceFile SuiteFolder SuiteFolder Exactly which is returned depends on whether it s a file or a folder.", "response": "def RobotFactory(path, parent=None):\n    '''Return an instance of SuiteFile, ResourceFile, SuiteFolder\n\n    Exactly which is returned depends on whether it's a file or\n    folder, and if a file, the contents of the file. If there is a\n    testcase table, this will return an instance of SuiteFile,\n    otherwise it will return an instance of ResourceFile.\n    '''\n\n    if os.path.isdir(path):\n        return SuiteFolder(path, parent)\n\n    else:\n        rf = RobotFile(path, parent)\n\n        for table in rf.tables:\n            if isinstance(table, TestcaseTable):\n                rf.__class__ = SuiteFile\n                return rf\n\n        rf.__class__ = ResourceFile\n        return rf"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef walk(self, *types):\n        '''\n        Iterator which visits all suites and suite files,\n        yielding test cases and keywords\n        '''\n        requested = types if len(types) > 0 else [SuiteFile, ResourceFile, SuiteFolder, Testcase, Keyword]\n\n        for thing in self.robot_files:\n            if thing.__class__ in requested:\n                yield thing\n            if isinstance(thing, SuiteFolder):\n                for child in thing.walk():\n                    if child.__class__ in requested:\n                        yield child\n            else:\n                for child in thing.walk(*types):\n                    yield child", "response": "Iterator which visits all suites and suite files and suite folders and test cases and keywords and yield test cases and keywords."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns a list of all folders and test suite files.", "response": "def robot_files(self):\n        '''Return a list of all folders, and test suite files (.txt, .robot)\n        '''\n        result = []\n        for name in os.listdir(self.path):\n            fullpath = os.path.join(self.path, name)\n            if os.path.isdir(fullpath):\n                result.append(RobotFactory(fullpath, parent=self))\n            else:\n                if ((name.endswith(\".txt\") or name.endswith(\".robot\")) and\n                    (name not in (\"__init__.txt\", \"__init__.robot\"))):\n\n                    result.append(RobotFactory(fullpath, parent=self))\n        return result"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nload the contents of the file at the given path and creates a list of tables and rows for each of the tables.", "response": "def _load(self, path):\n\n        '''\n        The general idea is to do a quick parse, creating a list of\n        tables. Each table is nothing more than a list of rows, with\n        each row being a list of cells. Additional parsing such as\n        combining rows into statements is done on demand. This first\n        pass is solely to read in the plain text and organize it by table.\n        '''\n\n        self.tables = []\n        current_table = DefaultTable(self)\n\n        with Utf8Reader(path) as f:\n            # N.B. the caller should be catching errors\n            self.raw_text = f.read()\n\n            f._file.seek(0) # bleh; wish this wasn't a private property\n            matcher = Matcher(re.IGNORECASE)\n            for linenumber, raw_text in enumerate(f.readlines()):\n                linenumber += 1; # start counting at 1 rather than zero\n\n                # this mimics what the robot TSV reader does --\n                # it replaces non-breaking spaces with regular spaces,\n                # and then strips trailing whitespace\n                raw_text = raw_text.replace(u'\\xA0', ' ')\n                raw_text = raw_text.rstrip()\n\n                # FIXME: I'm keeping line numbers but throwing away\n                # where each cell starts. I should be preserving that\n                # (though to be fair, robot is throwing that away so\n                # I'll have to write my own splitter if I want to save\n                # the character position)\n                cells = TxtReader.split_row(raw_text)\n                _heading_regex = r'^\\s*\\*+\\s*(.*?)[ *]*$'\n\n                if matcher(_heading_regex, cells[0]):\n                    # we've found the start of a new table\n                    table_name = matcher.group(1)\n                    current_table = tableFactory(self, linenumber, table_name, raw_text)\n                    self.tables.append(current_table)\n                else:\n                    current_table.append(Row(linenumber, raw_text, cells))"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns suite or resource if any of the tables are not UnknownTable", "response": "def type(self):\n        '''Return 'suite' or 'resource' or None\n\n        This will return 'suite' if a testcase table is found;\n        It will return 'resource' if at least one robot table\n        is found. If no tables are found it will return None\n        '''\n\n        robot_tables = [table for table in self.tables if not isinstance(table, UnknownTable)]\n        if len(robot_tables) == 0:\n            return None\n\n        for table in self.tables:\n            if isinstance(table, TestcaseTable):\n                return \"suite\"\n\n        return \"resource\""}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef keywords(self):\n        '''Generator which returns all keywords in the suite'''\n        for table in self.tables:\n            if isinstance(table, KeywordTable):\n                for keyword in table.keywords:\n                    yield keyword", "response": "Generator which returns all keywords in the suite"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ndump the tables and rows", "response": "def dump(self):\n        '''Regurgitate the tables and rows'''\n        for table in self.tables:\n            print(\"*** %s ***\" % table.name)\n            table.dump()"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef statements(self):\n        '''Return a list of statements\n\n        This is done by joining together any rows that\n        have continuations\n        '''\n        # FIXME: no need to do this every time; we should cache the\n        # result\n        if len(self.rows) == 0:\n            return []\n\n        current_statement = Statement(self.rows[0])\n        current_statement.startline = self.rows[0].linenumber\n        current_statement.endline = self.rows[0].linenumber\n        statements = []\n        for row in self.rows[1:]:\n            if len(row) > 0 and row[0] == \"...\":\n                # we found a continuation\n                current_statement += row[1:]\n                current_statement.endline = row.linenumber\n            else:\n                if len(current_statement) > 0:\n                    # append current statement to the list of statements...\n                    statements.append(current_statement)\n                # start a new statement\n                current_statement = Statement(row)\n                current_statement.startline = row.linenumber\n                current_statement.endline = row.linenumber\n\n        if len(current_statement) > 0:\n            statements.append(current_statement)\n\n        # trim trailing blank statements\n        while (len(statements[-1]) == 0 or \n               ((len(statements[-1]) == 1) and len(statements[-1][0]) == 0)):\n            statements.pop()\n        return statements", "response": "Return a list of statements that are part of the table. This is done by joining together any rows that have continuations\n       "}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nappends a new entry to the list of testcases.", "response": "def append(self, row):\n        ''' \n        The idea is, we recognize when we have a new testcase by \n        checking the first cell. If it's not empty and not a comment, \n        we have a new test case.\n\n        '''\n        if len(row) == 0:\n            # blank line. Should we throw it away, or append a BlankLine object?\n            return\n\n        if (row[0] != \"\" and \n            (not row[0].lstrip().startswith(\"#\"))):\n            # we have a new child table\n            self._children.append(self._childClass(self.parent, row.linenumber, row[0]))\n            if len(row.cells) > 1:\n                # It appears the first row -- which contains the test case or\n                # keyword name -- also has the first logical row of cells.\n                # We'll create a Row, but we'll make the first cell empty instead\n                # of leaving the name in it, since other code always assumes the\n                # first cell is empty. \n                #\n                # To be honest, I'm not sure this is the Right Thing To Do, but \n                # I'm too lazy to audit the code to see if it matters if we keep \n                # the first cell intact. Sorry if this ends up causing you grief\n                # some day...\n                row[0] = \"\"\n                self._children[-1].append(row.linenumber, row.raw_text, row.cells)\n\n        elif len(self._children) == 0:\n            # something before the first test case\n            # For now, append it to self.comments; eventually we should flag\n            # an error if it's NOT a comment\n            self.comments.append(row)\n\n        else:\n            # another row for the testcase\n            if len(row.cells) > 0:\n                self._children[-1].append(row.linenumber, row.raw_text, row.cells)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreporting an error or warning", "response": "def report(self, obj, message, linenum, char_offset=0):\n        \"\"\"Report an error or warning\"\"\"\n        self.controller.report(linenumber=linenum, filename=obj.path,\n                               severity=self.severity, message=message,\n                               rulename = self.__class__.__name__,\n                               char=char_offset)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the docstrings of the entry.", "response": "def doc(self):\n        '''Algorithm from https://www.python.org/dev/peps/pep-0257/'''\n        if not self.__doc__:\n            return \"\"\n\n        lines = self.__doc__.expandtabs().splitlines()\n\n        # Determine minimum indentation (first line doesn't count):\n        indent = sys.maxsize\n        for line in lines[1:]:\n            stripped = line.lstrip()\n            if stripped:\n                indent = min(indent, len(line) - len(stripped))\n\n        # Remove indentation (first line is special):\n        trimmed = [lines[0].strip()]\n        if indent < sys.maxsize:\n            for line in lines[1:]:\n                trimmed.append(line[indent:].rstrip())\n\n        # Strip off trailing and leading blank lines:\n        while trimmed and not trimmed[-1]:\n            trimmed.pop()\n        while trimmed and not trimmed[0]:\n            trimmed.pop(0)\n\n        # Return a single string:\n        return '\\n'.join(trimmed)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef run(self, args):\n\n        self.args = self.parse_and_process_args(args)\n\n        if self.args.version:\n            print(__version__)\n            return 0\n            \n        if self.args.rulefile:\n            for filename in self.args.rulefile:\n                self._load_rule_file(filename)\n\n        if self.args.list:\n            self.list_rules()\n            return 0\n        \n        if self.args.describe:\n            self._describe_rules(self.args.args)\n            return 0\n\n        self.counts = { ERROR: 0, WARNING: 0, \"other\": 0}\n            \n        for filename in self.args.args:\n            if not (os.path.exists(filename)):\n                sys.stderr.write(\"rflint: %s: No such file or directory\\n\" % filename)\n                continue\n            if os.path.isdir(filename):\n                self._process_folder(filename)\n            else:\n                self._process_file(filename)\n\n        if self.counts[ERROR] > 0:\n            return self.counts[ERROR] if self.counts[ERROR] < 254 else 255\n\n        return 0", "response": "Parse command line arguments and run rflint"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef list_rules(self):\n        for rule in sorted(self.all_rules, key=lambda rule: rule.name):\n            print(rule)\n            if self.args.verbose:\n                for line in rule.doc.split(\"\\n\"):\n                    print(\"    \", line)", "response": "Print a list of all rules"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef report(self, linenumber, filename, severity, message, rulename, char):\n\n        if self._print_filename is not None:\n            # we print the filename only once. self._print_filename\n            # will get reset each time a new file is processed.\n            print(\"+ \" + self._print_filename)\n            self._print_filename = None\n\n        if severity in (WARNING, ERROR):\n            self.counts[severity] += 1\n        else:\n            self.counts[\"other\"] += 1\n\n        print(self.args.format.format(linenumber=linenumber, filename=filename,\n                                      severity=severity, message=message.encode('utf-8'),\n                                      rulename=rulename, char=char))", "response": "Report a rule violation"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning a list of rules of a given class", "response": "def _get_rules(self, cls):\n        \"\"\"Returns a list of rules of a given class\n        \n        Rules are treated as singletons - we only instantiate each\n        rule once. \n        \"\"\"\n\n        result = []\n        for rule_class in cls.__subclasses__():\n            rule_name = rule_class.__name__.lower()\n            if rule_name not in self._rules:\n                rule = rule_class(self)\n                self._rules[rule_name] = rule\n            result.append(self._rules[rule_name])\n        return result"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _load_rule_file(self, filename):\n        if not (os.path.exists(filename)):\n            sys.stderr.write(\"rflint: %s: No such file or directory\\n\" % filename)\n            return\n        try:\n            basename = os.path.basename(filename)\n            (name, ext) = os.path.splitext(basename)\n            imp.load_source(name, filename)\n        except Exception as e:\n            sys.stderr.write(\"rflint: %s: exception while loading: %s\\n\" % (filename, str(e)))", "response": "Import the given rule file"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nhandle the parsing of command line arguments.", "response": "def parse_and_process_args(self, args):\n        \"\"\"Handle the parsing of command line arguments.\"\"\"\n\n        parser = argparse.ArgumentParser(\n            prog=\"python -m rflint\",\n            description=\"A style checker for robot framework plain text files.\",\n            formatter_class=argparse.RawDescriptionHelpFormatter,\n            epilog = (\n                \"You can use 'all' in place of RULENAME to refer to all rules. \\n\"\n                \"\\n\"\n                \"For example: '--ignore all --warn DuplicateTestNames' will ignore all\\n\"\n                \"rules except DuplicateTestNames.\\n\"\n                \"\\n\"\n                \"FORMAT is a string that performs a substitution on the following \\n\"\n                \"patterns: {severity}, {linenumber}, {char}, {message}, and {rulename}.\\n\"\n                \"\\n\"\n                \"For example: --format 'line: {linenumber}: message: {message}'. \\n\"\n                \"\\n\"\n                \"ARGUMENTFILE is a filename with contents that match the format of \\n\"\n                \"standard robot framework argument files\\n\"\n                \"\\n\"\n                \"If you give a directory as an argument, all files in the directory\\n\"\n                \"with the suffix .txt, .robot or .tsv will be processed. With the \\n\"\n                \"--recursive option, subfolders within the directory will also be\\n\"\n                \"processed.\"\n                )\n            )\n        parser.add_argument(\"--error\", \"-e\", metavar=\"RULENAME\", action=SetErrorAction,\n                            help=\"Assign a severity of ERROR to the given RULENAME\")\n        parser.add_argument(\"--ignore\", \"-i\", metavar=\"RULENAME\", action=SetIgnoreAction,\n                            help=\"Ignore the given RULENAME\")\n        parser.add_argument(\"--warning\", \"-w\", metavar=\"RULENAME\", action=SetWarningAction,\n                            help=\"Assign a severity of WARNING for the given RULENAME\")\n        parser.add_argument(\"--list\", \"-l\", action=\"store_true\",\n                            help=\"show a list of known rules and exit\")\n        parser.add_argument(\"--describe\", \"-d\", action=\"store_true\",\n                            help=\"describe the given rules\")\n        parser.add_argument(\"--no-filenames\", action=\"store_false\", dest=\"print_filenames\", \n                            default=True,\n                            help=\"suppress the printing of filenames\")\n        parser.add_argument(\"--format\", \"-f\", \n                            help=\"Define the output format\",\n                            default='{severity}: {linenumber}, {char}: {message} ({rulename})')\n        parser.add_argument(\"--version\", action=\"store_true\", default=False,\n                            help=\"Display version number and exit\")\n        parser.add_argument(\"--verbose\", \"-v\", action=\"store_true\", default=False,\n                            help=\"Give verbose output\")\n        parser.add_argument(\"--configure\", \"-c\", action=ConfigureAction,\n                            help=\"Configure a rule\")\n        parser.add_argument(\"--recursive\", \"-r\", action=\"store_true\", default=False,\n                            help=\"Recursively scan subfolders in a directory\")\n        parser.add_argument(\"--rulefile\", \"-R\", action=RulefileAction,\n                            help=\"import additional rules from the given RULEFILE\")\n        parser.add_argument(\"--argumentfile\", \"-A\", action=ArgfileLoader,\n                            help=\"read arguments from the given file\")\n        parser.add_argument('args', metavar=\"file\", nargs=argparse.REMAINDER)\n\n        # create a custom namespace, in which we can store a reference to\n        # our rules. This lets the custom argument actions access the list\n        # of rules\n        ns = argparse.Namespace()\n        setattr(ns, \"app\", self)\n        args = parser.parse_args(args, ns)\n\n        Rule.output_format = args.format\n\n        return args"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef from_resolver(cls, spec_resolver):\n        spec_validators = cls._get_spec_validators(spec_resolver)\n        return validators.extend(Draft4Validator, spec_validators)", "response": "Creates a customized Draft4ExtendedValidator."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating json documents validator from spec resolver.", "response": "def create(self, spec_resolver):\n        \"\"\"Creates json documents validator from spec resolver.\n        :param spec_resolver: reference resolver.\n\n        :return: RefResolver for spec with cached remote $refs used during\n            validation.\n        :rtype: :class:`jsonschema.RefResolver`\n        \"\"\"\n        validator_cls = self.spec_validator_factory.from_resolver(\n            spec_resolver)\n\n        return validator_cls(\n            self.schema, resolver=self.schema_resolver)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef construct_mapping(self, node, deep=False):\n        mapping = super(ExtendedSafeConstructor, self).construct_mapping(\n            node, deep)\n\n        return {\n            (str(key) if isinstance(key, int) else key): mapping[key]\n            for key in mapping\n        }", "response": "This method coerces all keys in the node to strings."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nopen a file and read it and return its contents.", "response": "def read_yaml_file(path, loader=ExtendedSafeLoader):\n    \"\"\"Open a file, read it and return its contents.\"\"\"\n    with open(path) as fh:\n        return load(fh, loader)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating validators generator for the spec resolver.", "response": "def from_spec_resolver(cls, spec_resolver):\n        \"\"\"Creates validators generator for the spec resolver.\n\n        :param spec_resolver: resolver for the spec\n        :type instance_resolver: :class:`jsonschema.RefResolver`\n        \"\"\"\n        deref = DerefValidatorDecorator(spec_resolver)\n        for key, validator_callable in iteritems(cls.validators):\n            yield key, deref(validator_callable)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef grade(grader_data,submission):\n\n    #Initialize result dictionary\n    results = {'errors': [],'tests': [],'score': 0, 'feedback' : \"\", 'success' : False, 'confidence' : 0}\n    has_error=False\n\n    grader_set=EssaySet(essaytype=\"test\")\n    feedback = {}\n\n    model, extractor = get_classifier_and_ext(grader_data)\n\n    #This is to preserve legacy functionality\n    if 'algorithm' not in grader_data:\n        grader_data['algorithm'] = util_functions.AlgorithmTypes.classification\n\n    try:\n        #Try to add essay to essay set object\n        grader_set.add_essay(str(submission),0)\n        grader_set.update_prompt(str(grader_data['prompt']))\n    except Exception:\n        error_message = \"Essay could not be added to essay set:{0}\".format(submission)\n        log.exception(error_message)\n        results['errors'].append(error_message)\n        has_error=True\n\n    #Try to extract features from submission and assign score via the model\n    try:\n        grader_feats=extractor.gen_feats(grader_set)\n        feedback=extractor.gen_feedback(grader_set,grader_feats)[0]\n        results['score']=int(model.predict(grader_feats)[0])\n    except Exception:\n        error_message = \"Could not extract features and score essay.\"\n        log.exception(error_message)\n        results['errors'].append(error_message)\n        has_error=True\n\n    #Try to determine confidence level\n    try:\n        results['confidence'] = get_confidence_value(grader_data['algorithm'], model, grader_feats, results['score'], grader_data['score'])\n    except Exception:\n        #If there is an error getting confidence, it is not a show-stopper, so just log\n        log.exception(\"Problem generating confidence value\")\n\n    if not has_error:\n\n        #If the essay is just a copy of the prompt, return a 0 as the score\n        if( 'too_similar_to_prompt' in feedback and feedback['too_similar_to_prompt']):\n            results['score']=0\n            results['correct']=False\n\n        results['success']=True\n\n        #Generate short form output--number of problem areas identified in feedback\n\n        #Add feedback to results if available\n        results['feedback'] = {}\n        if 'topicality' in feedback and 'prompt_overlap' in feedback:\n            results['feedback'].update({\n                'topicality' : feedback['topicality'],\n                'prompt-overlap' : feedback['prompt_overlap'],\n                })\n\n        results['feedback'].update(\n            {\n                'spelling' : feedback['spelling'],\n                'grammar' : feedback['grammar'],\n                'markup-text' : feedback['markup_text'],\n                }\n        )\n\n    else:\n        #If error, success is False.\n        results['success']=False\n\n    return results", "response": "Grades a specified submission using the specified models"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngrades a set of numeric and textual features using a generic model", "response": "def grade_generic(grader_data, numeric_features, textual_features):\n    \"\"\"\n    Grades a set of numeric and textual features using a generic model\n    grader_data -- dictionary containing:\n    {\n        'algorithm' - Type of algorithm to use to score\n    }\n    numeric_features - list of numeric features to predict on\n    textual_features - list of textual feature to predict on\n\n    \"\"\"\n    results = {'errors': [],'tests': [],'score': 0, 'success' : False, 'confidence' : 0}\n\n    has_error=False\n\n    #Try to find and load the model file\n\n    grader_set=predictor_set.PredictorSet(essaytype=\"test\")\n\n    model, extractor = get_classifier_and_ext(grader_data)\n\n    #Try to add essays to essay set object\n    try:\n        grader_set.add_row(numeric_features, textual_features,0)\n    except Exception:\n        error_msg = \"Row could not be added to predictor set:{0} {1}\".format(numeric_features, textual_features)\n        log.exception(error_msg)\n        results['errors'].append(error_msg)\n        has_error=True\n\n    #Try to extract features from submission and assign score via the model\n    try:\n        grader_feats=extractor.gen_feats(grader_set)\n        results['score']=model.predict(grader_feats)[0]\n    except Exception:\n        error_msg = \"Could not extract features and score essay.\"\n        log.exception(error_msg)\n        results['errors'].append(error_msg)\n        has_error=True\n\n    #Try to determine confidence level\n    try:\n        results['confidence'] = get_confidence_value(grader_data['algorithm'],model, grader_feats, results['score'])\n    except Exception:\n        #If there is an error getting confidence, it is not a show-stopper, so just log\n        log.exception(\"Problem generating confidence value\")\n\n    if not has_error:\n        results['success'] = True\n\n    return results"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_confidence_value(algorithm,model,grader_feats,score, scores):\n    min_score=min(numpy.asarray(scores))\n    max_score=max(numpy.asarray(scores))\n    if algorithm == util_functions.AlgorithmTypes.classification and hasattr(model, \"predict_proba\"):\n        #If classification, predict with probability, which gives you a matrix of confidences per score point\n        raw_confidence=model.predict_proba(grader_feats)[0,(float(score)-float(min_score))]\n        #TODO: Normalize confidence somehow here\n        confidence=raw_confidence\n    elif hasattr(model, \"predict\"):\n        raw_confidence = model.predict(grader_feats)[0]\n        confidence = max(float(raw_confidence) - math.floor(float(raw_confidence)), math.ceil(float(raw_confidence)) - float(raw_confidence))\n    else:\n        confidence = 0\n\n    return confidence", "response": "Determines a confidence value given a proper input parameters"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef create_model_path(model_path):\n    if not model_path.startswith(\"/\") and not model_path.startswith(\"models/\"):\n        model_path=\"/\" + model_path\n    if not model_path.startswith(\"models\"):\n        model_path = \"models\" + model_path\n    if not model_path.endswith(\".p\"):\n        model_path+=\".p\"\n\n    return model_path", "response": "Creates a path to model files based on the model path."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef sub_chars(string):\n    #Define replacement patterns\n    sub_pat = r\"[^A-Za-z\\.\\?!,';:]\"\n    char_pat = r\"\\.\"\n    com_pat = r\",\"\n    ques_pat = r\"\\?\"\n    excl_pat = r\"!\"\n    sem_pat = r\";\"\n    col_pat = r\":\"\n    whitespace_pat = r\"\\s{1,}\"\n\n    #Replace text.  Ordering is very important!\n    nstring = re.sub(sub_pat, \" \", string)\n    nstring = re.sub(char_pat,\" .\", nstring)\n    nstring = re.sub(com_pat, \" ,\", nstring)\n    nstring = re.sub(ques_pat, \" ?\", nstring)\n    nstring = re.sub(excl_pat, \" !\", nstring)\n    nstring = re.sub(sem_pat, \" ;\", nstring)\n    nstring = re.sub(col_pat, \" :\", nstring)\n    nstring = re.sub(whitespace_pat, \" \", nstring)\n\n    return nstring", "response": "Strips illegal characters from a string. Used to sanitize input essays."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef spell_correct(string):\n\n    # Create a temp file so that aspell could be used\n    # By default, tempfile will delete this file when the file handle is closed.\n    f = tempfile.NamedTemporaryFile(mode='w')\n    f.write(string)\n    f.flush()\n    f_path = os.path.abspath(f.name)\n    try:\n        p = os.popen(aspell_path + \" -a < \" + f_path + \" --sug-mode=ultra\")\n\n        # Aspell returns a list of incorrect words with the above flags\n        incorrect = p.readlines()\n        p.close()\n\n    except Exception:\n        log.exception(\"aspell process failed; could not spell check\")\n        # Return original string if aspell fails\n        return string,0, string\n\n    finally:\n        f.close()\n\n    incorrect_words = list()\n    correct_spelling = list()\n    for i in range(1, len(incorrect)):\n        if(len(incorrect[i]) > 10):\n            #Reformat aspell output to make sense\n            match = re.search(\":\", incorrect[i])\n            if hasattr(match, \"start\"):\n                begstring = incorrect[i][2:match.start()]\n                begmatch = re.search(\" \", begstring)\n                begword = begstring[0:begmatch.start()]\n\n                sugstring = incorrect[i][match.start() + 2:]\n                sugmatch = re.search(\",\", sugstring)\n                if hasattr(sugmatch, \"start\"):\n                    sug = sugstring[0:sugmatch.start()]\n\n                    incorrect_words.append(begword)\n                    correct_spelling.append(sug)\n\n    #Create markup based on spelling errors\n    newstring = string\n    markup_string = string\n    already_subbed=[]\n    for i in range(0, len(incorrect_words)):\n        sub_pat = r\"\\b\" + incorrect_words[i] + r\"\\b\"\n        sub_comp = re.compile(sub_pat)\n        newstring = re.sub(sub_comp, correct_spelling[i], newstring)\n        if incorrect_words[i] not in already_subbed:\n            markup_string=re.sub(sub_comp,'<bs>' + incorrect_words[i] + \"</bs>\", markup_string)\n            already_subbed.append(incorrect_words[i])\n\n    return newstring,len(incorrect_words),markup_string", "response": "This function will spell correct an input string."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ngenerate ngrams from a list of words.", "response": "def ngrams(tokens, min_n, max_n):\n    \"\"\"\n    Generates ngrams(word sequences of fixed length) from an input token sequence.\n    tokens is a list of words.\n    min_n is the minimum length of an ngram to return.\n    max_n is the maximum length of an ngram to return.\n    returns a list of ngrams (words separated by a space)\n    \"\"\"\n    all_ngrams = list()\n    n_tokens = len(tokens)\n    for i in xrange(n_tokens):\n        for j in xrange(i + min_n, min(n_tokens, i + max_n) + 1):\n            all_ngrams.append(\" \".join(tokens[i:j]))\n    return all_ngrams"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef f7(seq):\n    seen = set()\n    seen_add = seen.add\n    return [x for x in seq if x not in seen and not seen_add(x)]", "response": "Makes a list unique\n    Makes a list unique\n    Makes a list unique\n    Makes a list unique\n    Makes a list unique\n    Makes a list unique\n    Makes a list unique\n    Makes a list unique\n    Makes a list unique\n    Makes a list unique\n    Makes a list unique\n    Makes a list unique\n    Makes a list unique\n    Makes a list unique\n   "}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ngenerating a count of the number of times each unique item appears in a list", "response": "def count_list(the_list):\n    \"\"\"\n    Generates a count of the number of times each unique item appears in a list\n    \"\"\"\n    count = the_list.count\n    result = [(item, count(item)) for item in set(the_list)]\n    result.sort()\n    return result"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ngenerating a list of ngrams that appear in the string.", "response": "def regenerate_good_tokens(string):\n    \"\"\"\n    Given an input string, part of speech tags the string, then generates a list of\n    ngrams that appear in the string.\n    Used to define grammatically correct part of speech tag sequences.\n    Returns a list of part of speech tag sequences.\n    \"\"\"\n    toks = nltk.word_tokenize(string)\n    pos_string = nltk.pos_tag(toks)\n    pos_seq = [tag[1] for tag in pos_string]\n    pos_ngrams = ngrams(pos_seq, 2, 4)\n    sel_pos_ngrams = f7(pos_ngrams)\n    return sel_pos_ngrams"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nusing a fisher test to find words that are significant in that they separate high scoring essays from low scoring essays. text is a list of input essays. score is a list of scores, with score[n] corresponding to text[n] max_feats is the maximum number of features to consider in the first pass max_feats2 is the maximum number of features to consider in the second (final) pass Returns a list of words that constitute the significant vocabulary", "response": "def get_vocab(text, score, max_feats=750, max_feats2=200):\n    \"\"\"\n    Uses a fisher test to find words that are significant in that they separate\n    high scoring essays from low scoring essays.\n    text is a list of input essays.\n    score is a list of scores, with score[n] corresponding to text[n]\n    max_feats is the maximum number of features to consider in the first pass\n    max_feats2 is the maximum number of features to consider in the second (final) pass\n    Returns a list of words that constitute the significant vocabulary\n    \"\"\"\n    dict = CountVectorizer(ngram_range=(1,2), max_features=max_feats)\n    dict_mat = dict.fit_transform(text)\n    set_score = numpy.asarray(score, dtype=numpy.int)\n    med_score = numpy.median(set_score)\n    new_score = set_score\n    if(med_score == 0):\n        med_score = 1\n    new_score[set_score < med_score] = 0\n    new_score[set_score >= med_score] = 1\n\n    fish_vals = []\n    for col_num in range(0, dict_mat.shape[1]):\n        loop_vec = dict_mat.getcol(col_num).toarray()\n        good_loop_vec = loop_vec[new_score == 1]\n        bad_loop_vec = loop_vec[new_score == 0]\n        good_loop_present = len(good_loop_vec[good_loop_vec > 0])\n        good_loop_missing = len(good_loop_vec[good_loop_vec == 0])\n        bad_loop_present = len(bad_loop_vec[bad_loop_vec > 0])\n        bad_loop_missing = len(bad_loop_vec[bad_loop_vec == 0])\n        fish_val = pvalue(good_loop_present, bad_loop_present, good_loop_missing, bad_loop_missing).two_tail\n        fish_vals.append(fish_val)\n\n    cutoff = 1\n    if(len(fish_vals) > max_feats2):\n        cutoff = sorted(fish_vals)[max_feats2]\n    good_cols = numpy.asarray([num for num in range(0, dict_mat.shape[1]) if fish_vals[num] <= cutoff])\n\n    getVar = lambda searchList, ind: [searchList[i] for i in ind]\n    vocab = getVar(dict.get_feature_names(), good_cols)\n\n    return vocab"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncalculate edit distance between strings 1 and string 2.", "response": "def edit_distance(s1, s2):\n    \"\"\"\n    Calculates string edit distance between string 1 and string 2.\n    Deletion, insertion, substitution, and transposition all increase edit distance.\n    \"\"\"\n    d = {}\n    lenstr1 = len(s1)\n    lenstr2 = len(s2)\n    for i in xrange(-1, lenstr1 + 1):\n        d[(i, -1)] = i + 1\n    for j in xrange(-1, lenstr2 + 1):\n        d[(-1, j)] = j + 1\n\n    for i in xrange(lenstr1):\n        for j in xrange(lenstr2):\n            if s1[i] == s2[j]:\n                cost = 0\n            else:\n                cost = 1\n            d[(i, j)] = min(\n                d[(i - 1, j)] + 1, # deletion\n                d[(i, j - 1)] + 1, # insertion\n                d[(i - 1, j - 1)] + cost, # substitution\n            )\n            if i and j and s1[i] == s2[j - 1] and s1[i - 1] == s2[j]:\n                d[(i, j)] = min(d[(i, j)], d[i - 2, j - 2] + cost) # transposition\n\n    return d[lenstr1 - 1, lenstr2 - 1]"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngenerating cross validated predictions using an input classifier and data.", "response": "def gen_cv_preds(clf, arr, sel_score, num_chunks=3):\n    \"\"\"\n    Generates cross validated predictions using an input classifier and data.\n    clf is a classifier that implements that implements the fit and predict methods.\n    arr is the input data array (X)\n    sel_score is the target list (y).  y[n] corresponds to X[n,:]\n    num_chunks is the number of cross validation folds to use\n    Returns an array of the predictions where prediction[n] corresponds to X[n,:]\n    \"\"\"\n    cv_len = int(math.floor(len(sel_score) / num_chunks))\n    chunks = []\n    for i in range(0, num_chunks):\n        range_min = i * cv_len\n        range_max = ((i + 1) * cv_len)\n        if i == num_chunks - 1:\n            range_max = len(sel_score)\n        chunks.append(range(range_min, range_max))\n    preds = []\n    set_score = numpy.asarray(sel_score, dtype=numpy.int)\n    chunk_vec = numpy.asarray(range(0, len(chunks)))\n    for i in xrange(0, len(chunks)):\n        loop_inds = list(\n            chain.from_iterable([chunks[int(z)] for z, m in enumerate(range(0, len(chunks))) if int(z) != i]))\n        sim_fit = clf.fit(arr[loop_inds], set_score[loop_inds])\n        preds.append(list(sim_fit.predict(arr[chunks[i]])))\n    all_preds = list(chain(*preds))\n    return(all_preds)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef gen_model(clf, arr, sel_score):\n    set_score = numpy.asarray(sel_score, dtype=numpy.int)\n    sim_fit = clf.fit(arr, set_score)\n    return(sim_fit)", "response": "Generates a model from a classifier to data and a target score."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef gen_preds(clf, arr):\n    if(hasattr(clf, \"predict_proba\")):\n        ret = clf.predict(arr)\n        # pred_score=preds.argmax(1)+min(x._score)\n    else:\n        ret = clf.predict(arr)\n    return ret", "response": "Generates predictions on a novel data array using a fit classifier clf and arr"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating the average value of a list of numbers Returns a float", "response": "def calc_list_average(l):\n    \"\"\"\n    Calculates the average value of a list of numbers\n    Returns a float\n    \"\"\"\n    total = 0.0\n    for value in l:\n        total += value\n    return total / len(l)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef quadratic_weighted_kappa(rater_a, rater_b, min_rating=None, max_rating=None):\n    assert(len(rater_a) == len(rater_b))\n    rater_a = [int(a) for a in rater_a]\n    rater_b = [int(b) for b in rater_b]\n    if min_rating is None:\n        min_rating = min(rater_a + rater_b)\n    if max_rating is None:\n        max_rating = max(rater_a + rater_b)\n    conf_mat = confusion_matrix(rater_a, rater_b,\n        min_rating, max_rating)\n    num_ratings = len(conf_mat)\n    num_scored_items = float(len(rater_a))\n\n    hist_rater_a = histogram(rater_a, min_rating, max_rating)\n    hist_rater_b = histogram(rater_b, min_rating, max_rating)\n\n    numerator = 0.0\n    denominator = 0.0\n\n    if(num_ratings > 1):\n        for i in range(num_ratings):\n            for j in range(num_ratings):\n                expected_count = (hist_rater_a[i] * hist_rater_b[j]\n                                  / num_scored_items)\n                d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)\n                numerator += d * conf_mat[i][j] / num_scored_items\n                denominator += d * expected_count / num_scored_items\n\n        return 1.0 - numerator / denominator\n    else:\n        return 1.0", "response": "Calculates the kappa correlation between rater_a and rater_b."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):\n    assert(len(rater_a) == len(rater_b))\n    rater_a = [int(a) for a in rater_a]\n    rater_b = [int(b) for b in rater_b]\n    min_rating = int(min_rating)\n    max_rating = int(max_rating)\n    if min_rating is None:\n        min_rating = min(rater_a)\n    if max_rating is None:\n        max_rating = max(rater_a)\n    num_ratings = int(max_rating - min_rating + 1)\n    conf_mat = [[0 for i in range(num_ratings)]\n                for j in range(num_ratings)]\n    for a, b in zip(rater_a, rater_b):\n        conf_mat[int(a - min_rating)][int(b - min_rating)] += 1\n    return conf_mat", "response": "Generates a confusion matrix between rater_a and rater_b"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef histogram(ratings, min_rating=None, max_rating=None):\n    ratings = [int(r) for r in ratings]\n    if min_rating is None:\n        min_rating = min(ratings)\n    if max_rating is None:\n        max_rating = max(ratings)\n    num_ratings = int(max_rating - min_rating + 1)\n    hist_ratings = [0 for x in range(num_ratings)]\n    for r in ratings:\n        hist_ratings[r - min_rating] += 1\n    return hist_ratings", "response": "Generates a frequency count of each rating on the scale\n    ratings is a list of scores\n   "}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nutilize wordnet to get synonyms for words word is the input word returns a list of unique synonyms", "response": "def get_wordnet_syns(word):\n    \"\"\"\n    Utilize wordnet (installed with nltk) to get synonyms for words\n    word is the input word\n    returns a list of unique synonyms\n    \"\"\"\n    synonyms = []\n    regex = r\"_\"\n    pat = re.compile(regex)\n    synset = nltk.wordnet.wordnet.synsets(word)\n    for ss in synset:\n        for swords in ss.lemma_names:\n            synonyms.append(pat.sub(\" \", swords.lower()))\n    synonyms = f7(synonyms)\n    return synonyms"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_separator_words(toks1):\n    tab_toks1 = nltk.FreqDist(word.lower() for word in toks1)\n    if(os.path.isfile(ESSAY_COR_TOKENS_PATH)):\n        toks2 = pickle.load(open(ESSAY_COR_TOKENS_PATH, 'rb'))\n    else:\n        essay_corpus = open(ESSAY_CORPUS_PATH).read()\n        essay_corpus = sub_chars(essay_corpus)\n        toks2 = nltk.FreqDist(word.lower() for word in nltk.word_tokenize(essay_corpus))\n        pickle.dump(toks2, open(ESSAY_COR_TOKENS_PATH, 'wb'))\n    sep_words = []\n    for word in tab_toks1.keys():\n        tok1_present = tab_toks1[word]\n        if(tok1_present > 2):\n            tok1_total = tab_toks1._N\n            tok2_present = toks2[word]\n            tok2_total = toks2._N\n            fish_val = pvalue(tok1_present, tok2_present, tok1_total, tok2_total).two_tail\n            if(fish_val < .001 and tok1_present / float(tok1_total) > (tok2_present / float(tok2_total)) * 2):\n                sep_words.append(word)\n    sep_words = [w for w in sep_words if not w in nltk.corpus.stopwords.words(\"english\") and len(w) > 5]\n    return sep_words", "response": "Finds the words that separate a list of tokens from a background corpus\n    Returns a list of separator words"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getMedian(numericValues):\n    theValues = sorted(numericValues)\n\n    if len(theValues) % 2 == 1:\n        return theValues[(len(theValues) + 1) / 2 - 1]\n    else:\n        lower = theValues[len(theValues) / 2 - 1]\n        upper = theValues[len(theValues) / 2]\n\n        return (float(lower + upper)) / 2", "response": "Gets the median of a list of values Returns a float"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ninitialize the dictionary objects for the current object based on the input essay set e_set.", "response": "def initialize_dictionaries(self, e_set, max_feats2 = 200):\n        \"\"\"\n        Initializes dictionaries from an essay set object\n        Dictionaries must be initialized prior to using this to extract features\n        e_set is an input essay set\n        returns a confirmation of initialization\n        \"\"\"\n        if(hasattr(e_set, '_type')):\n            if(e_set._type == \"train\"):\n                #normal text (unstemmed) useful words/bigrams\n                nvocab = util_functions.get_vocab(e_set._text, e_set._score, max_feats2 = max_feats2)\n                #stemmed and spell corrected vocab useful words/ngrams\n                svocab = util_functions.get_vocab(e_set._clean_stem_text, e_set._score, max_feats2 = max_feats2)\n                #dictionary trained on proper vocab\n                self._normal_dict = CountVectorizer(ngram_range=(1,2), vocabulary=nvocab)\n                #dictionary trained on proper vocab\n                self._stem_dict = CountVectorizer(ngram_range=(1,2), vocabulary=svocab)\n                self.dict_initialized = True\n                #Average spelling errors in set. needed later for spelling detection\n                self._mean_spelling_errors=sum(e_set._spelling_errors)/float(len(e_set._spelling_errors))\n                self._spell_errors_per_character=sum(e_set._spelling_errors)/float(sum([len(t) for t in e_set._text]))\n                #Gets the number and positions of grammar errors\n                good_pos_tags,bad_pos_positions=self._get_grammar_errors(e_set._pos,e_set._text,e_set._tokens)\n                self._grammar_errors_per_character=(sum(good_pos_tags)/float(sum([len(t) for t in e_set._text])))\n                #Generate bag of words features\n                bag_feats=self.gen_bag_feats(e_set)\n                #Sum of a row of bag of words features (topical words in an essay)\n                f_row_sum=numpy.sum(bag_feats[:,:])\n                #Average index of how \"topical\" essays are\n                self._mean_f_prop=f_row_sum/float(sum([len(t) for t in e_set._text]))\n                ret = \"ok\"\n            else:\n                raise util_functions.InputError(e_set, \"needs to be an essay set of the train type.\")\n        else:\n            raise util_functions.InputError(e_set, \"wrong input. need an essay set object\")\n        return ret"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef get_good_pos_ngrams(self):\n        if(os.path.isfile(NGRAM_PATH)):\n            good_pos_ngrams = pickle.load(open(NGRAM_PATH, 'rb'))\n        elif os.path.isfile(ESSAY_CORPUS_PATH):\n            essay_corpus = open(ESSAY_CORPUS_PATH).read()\n            essay_corpus = util_functions.sub_chars(essay_corpus)\n            good_pos_ngrams = util_functions.regenerate_good_tokens(essay_corpus)\n            pickle.dump(good_pos_ngrams, open(NGRAM_PATH, 'wb'))\n        else:\n            #Hard coded list in case the needed files cannot be found\n            good_pos_ngrams=['NN PRP', 'NN PRP .', 'NN PRP . DT', 'PRP .', 'PRP . DT', 'PRP . DT NNP', '. DT',\n             '. DT NNP', '. DT NNP NNP', 'DT NNP', 'DT NNP NNP', 'DT NNP NNP NNP', 'NNP NNP',\n             'NNP NNP NNP', 'NNP NNP NNP NNP', 'NNP NNP NNP .', 'NNP NNP .', 'NNP NNP . TO',\n             'NNP .', 'NNP . TO', 'NNP . TO NNP', '. TO', '. TO NNP', '. TO NNP NNP',\n             'TO NNP', 'TO NNP NNP']\n\n        return set(good_pos_ngrams)", "response": "Gets a set of gramatically correct part of speech sequences from an input file called essaycorpus. txt"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _get_grammar_errors(self,pos,text,tokens):\n        word_counts = [max(len(t),1) for t in tokens]\n        good_pos_tags = []\n        min_pos_seq=2\n        max_pos_seq=4\n        bad_pos_positions=[]\n        for i in xrange(0, len(text)):\n            pos_seq = [tag[1] for tag in pos[i]]\n            pos_ngrams = util_functions.ngrams(pos_seq, min_pos_seq, max_pos_seq)\n            long_pos_ngrams=[z for z in pos_ngrams if z.count(' ')==(max_pos_seq-1)]\n            bad_pos_tuples=[[z,z+max_pos_seq] for z in xrange(0,len(long_pos_ngrams)) if long_pos_ngrams[z] not in self._good_pos_ngrams]\n            bad_pos_tuples.sort(key=operator.itemgetter(1))\n            to_delete=[]\n            for m in reversed(xrange(len(bad_pos_tuples)-1)):\n                start, end = bad_pos_tuples[m]\n                for j in xrange(m+1, len(bad_pos_tuples)):\n                    lstart, lend = bad_pos_tuples[j]\n                    if lstart >= start and lstart <= end:\n                        bad_pos_tuples[m][1]=bad_pos_tuples[j][1]\n                        to_delete.append(j)\n\n            fixed_bad_pos_tuples=[bad_pos_tuples[z] for z in xrange(0,len(bad_pos_tuples)) if z not in to_delete]\n            bad_pos_positions.append(fixed_bad_pos_tuples)\n            overlap_ngrams = [z for z in pos_ngrams if z in self._good_pos_ngrams]\n            if (len(pos_ngrams)-len(overlap_ngrams))>0:\n                divisor=len(pos_ngrams)/len(pos_seq)\n            else:\n                divisor=1\n            if divisor == 0:\n                divisor=1\n            good_grammar_ratio = (len(pos_ngrams)-len(overlap_ngrams))/divisor\n            good_pos_tags.append(good_grammar_ratio)\n        return good_pos_tags,bad_pos_positions", "response": "Internal function to get the number of grammar errors in given text."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate length based features from an essay set containing only the words in the object.", "response": "def gen_length_feats(self, e_set):\n        \"\"\"\n        Generates length based features from an essay set\n        Generally an internal function called by gen_feats\n        Returns an array of length features\n        e_set - EssaySet object\n        \"\"\"\n        text = e_set._text\n        lengths = [len(e) for e in text]\n        word_counts = [max(len(t),1) for t in e_set._tokens]\n        comma_count = [e.count(\",\") for e in text]\n        ap_count = [e.count(\"'\") for e in text]\n        punc_count = [e.count(\".\") + e.count(\"?\") + e.count(\"!\") for e in text]\n        chars_per_word = [lengths[m] / float(word_counts[m]) for m in xrange(0, len(text))]\n\n        good_pos_tags,bad_pos_positions= self._get_grammar_errors(e_set._pos,e_set._text,e_set._tokens)\n        good_pos_tag_prop = [good_pos_tags[m] / float(word_counts[m]) for m in xrange(0, len(text))]\n\n        length_arr = numpy.array((\n        lengths, word_counts, comma_count, ap_count, punc_count, chars_per_word, good_pos_tags,\n        good_pos_tag_prop)).transpose()\n\n        return length_arr.copy()"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ngenerating bag of words features from an input essay set and trained FeatureExtractor object.", "response": "def gen_bag_feats(self, e_set):\n        \"\"\"\n        Generates bag of words features from an input essay set and trained FeatureExtractor\n        Generally called by gen_feats\n        Returns an array of features\n        e_set - EssaySet object\n        \"\"\"\n        if(hasattr(self, '_stem_dict')):\n            sfeats = self._stem_dict.transform(e_set._clean_stem_text)\n            nfeats = self._normal_dict.transform(e_set._text)\n            bag_feats = numpy.concatenate((sfeats.toarray(), nfeats.toarray()), axis=1)\n        else:\n            raise util_functions.InputError(self, \"Dictionaries must be initialized prior to generating bag features.\")\n        return bag_feats.copy()"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef gen_feats(self, e_set):\n        bag_feats = self.gen_bag_feats(e_set)\n        length_feats = self.gen_length_feats(e_set)\n        prompt_feats = self.gen_prompt_feats(e_set)\n        overall_feats = numpy.concatenate((length_feats, prompt_feats, bag_feats), axis=1)\n        overall_feats = overall_feats.copy()\n\n        return overall_feats", "response": "Generates bag of words length and prompt features from an essay set object"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef gen_prompt_feats(self, e_set):\n        prompt_toks = nltk.word_tokenize(e_set._prompt)\n        expand_syns = []\n        for word in prompt_toks:\n            synonyms = util_functions.get_wordnet_syns(word)\n            expand_syns.append(synonyms)\n        expand_syns = list(chain.from_iterable(expand_syns))\n        prompt_overlap = []\n        prompt_overlap_prop = []\n        for j in e_set._tokens:\n            tok_length=len(j)\n            if(tok_length==0):\n                tok_length=1\n            prompt_overlap.append(len([i for i in j if i in prompt_toks]))\n            prompt_overlap_prop.append(prompt_overlap[len(prompt_overlap) - 1] / float(tok_length))\n        expand_overlap = []\n        expand_overlap_prop = []\n        for j in e_set._tokens:\n            tok_length=len(j)\n            if(tok_length==0):\n                tok_length=1\n            expand_overlap.append(len([i for i in j if i in expand_syns]))\n            expand_overlap_prop.append(expand_overlap[len(expand_overlap) - 1] / float(tok_length))\n\n        prompt_arr = numpy.array((prompt_overlap, prompt_overlap_prop, expand_overlap, expand_overlap_prop)).transpose()\n\n        return prompt_arr.copy()", "response": "Generates prompt based features from an essay set object and internal prompt variable."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ngenerates a list of lists of feedback for a given set of essays.", "response": "def gen_feedback(self, e_set, features=None):\n        \"\"\"\n        Generate feedback for a given set of essays\n        e_set - EssaySet object\n        features - optionally, pass in a matrix of features extracted from e_set using FeatureExtractor\n        in order to get off topic feedback.\n        Returns a list of lists (one list per essay in e_set)\n        e_set - EssaySet object\n        \"\"\"\n\n        #Set ratio to modify thresholds for grammar/spelling errors\n        modifier_ratio=1.05\n\n        #Calc number of grammar and spelling errors per character\n        set_grammar,bad_pos_positions=self._get_grammar_errors(e_set._pos,e_set._text,e_set._tokens)\n        set_grammar_per_character=[set_grammar[m]/float(len(e_set._text[m])+.1) for m in xrange(0,len(e_set._text))]\n        set_spell_errors_per_character=[e_set._spelling_errors[m]/float(len(e_set._text[m])+.1) for m in xrange(0,len(e_set._text))]\n\n        #Iterate through essays and create a feedback dict for each\n        all_feedback=[]\n        for m in xrange(0,len(e_set._text)):\n            #Be very careful about changing these messages!\n            individual_feedback={'grammar' : \"Grammar: Ok.\",\n                                 'spelling' : \"Spelling: Ok.\",\n                                 'markup_text' : \"\",\n                                 'grammar_per_char' : set_grammar_per_character[m],\n                                 'spelling_per_char' : set_spell_errors_per_character[m],\n                                 'too_similar_to_prompt' : False,\n                                 }\n            markup_tokens=e_set._markup_text[m].split(\" \")\n\n            #This loop ensures that sequences of bad grammar get put together into one sequence instead of staying\n            #disjointed\n            bad_pos_starts=[z[0] for z in bad_pos_positions[m]]\n            bad_pos_ends=[z[1]-1 for z in bad_pos_positions[m]]\n            for z in xrange(0,len(markup_tokens)):\n                if z in bad_pos_starts:\n                    markup_tokens[z]='<bg>' + markup_tokens[z]\n                elif z in bad_pos_ends:\n                    markup_tokens[z]=markup_tokens[z] + \"</bg>\"\n            if(len(bad_pos_ends)>0 and len(bad_pos_starts)>0 and len(markup_tokens)>1):\n                if max(bad_pos_ends)>(len(markup_tokens)-1) and max(bad_pos_starts)<(len(markup_tokens)-1):\n                    markup_tokens[len(markup_tokens)-1]+=\"</bg>\"\n\n            #Display messages if grammar/spelling errors greater than average in training set\n            if set_grammar_per_character[m]>(self._grammar_errors_per_character*modifier_ratio):\n                individual_feedback['grammar']=\"Grammar: More grammar errors than average.\"\n            if set_spell_errors_per_character[m]>(self._spell_errors_per_character*modifier_ratio):\n                individual_feedback['spelling']=\"Spelling: More spelling errors than average.\"\n\n            #Test topicality by calculating # of on topic words per character and comparing to the training set\n            #mean.  Requires features to be passed in\n            if features is not None:\n                f_row_sum=numpy.sum(features[m,12:])\n                f_row_prop=f_row_sum/len(e_set._text[m])\n                if f_row_prop<(self._mean_f_prop/1.5) or len(e_set._text[m])<20:\n                    individual_feedback['topicality']=\"Topicality: Essay may be off topic.\"\n\n                if(features[m,9]>.6):\n                    individual_feedback['prompt_overlap']=\"Prompt Overlap: Too much overlap with prompt.\"\n                    individual_feedback['too_similar_to_prompt']=True\n                    log.debug(features[m,9])\n\n            #Create string representation of markup text\n            markup_string=\" \".join(markup_tokens)\n            individual_feedback['markup_text']=markup_string\n            all_feedback.append(individual_feedback)\n\n        return all_feedback"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nadds new essay to the set.", "response": "def add_essay(self, essay_text, essay_score, essay_generated=0):\n        \"\"\"\n        Add new (essay_text,essay_score) pair to the essay set.\n        essay_text must be a string.\n        essay_score must be an int.\n        essay_generated should not be changed by the user.\n        Returns a confirmation that essay was added.\n        \"\"\"\n        # Get maximum current essay id, or set to 0 if this is the first essay added\n        if(len(self._id) > 0):\n            max_id = max(self._id)\n        else:\n            max_id = 0\n            # Verify that essay_score is an int, essay_text is a string, and essay_generated equals 0 or 1\n\n        try:\n            essay_text = essay_text.encode('ascii', 'ignore')\n            if len(essay_text) < 5:\n                essay_text = \"Invalid essay.\"\n        except:\n            log.exception(\"Could not parse essay into ascii.\")\n\n        try:\n            # Try conversion of types\n            essay_score = int(essay_score)\n            essay_text = str(essay_text)\n        except:\n            # Nothing needed here, will return error in any case.\n            log.exception(\"Invalid type for essay score : {0} or essay text : {1}\".format(type(essay_score), type(essay_text)))\n\n        if isinstance(essay_score, int) and isinstance(essay_text, basestring)\\\n                and (essay_generated == 0 or essay_generated == 1):\n            self._id.append(max_id + 1)\n            self._score.append(essay_score)\n            # Clean text by removing non digit/work/punctuation characters\n            try:\n                essay_text = str(essay_text.encode('ascii', 'ignore'))\n            except:\n                essay_text = (essay_text.decode('utf-8', 'replace')).encode('ascii', 'ignore')\n            cleaned_essay = util_functions.sub_chars(essay_text).lower()\n            if(len(cleaned_essay) > MAXIMUM_ESSAY_LENGTH):\n                cleaned_essay = cleaned_essay[0:MAXIMUM_ESSAY_LENGTH]\n            self._text.append(cleaned_essay)\n            # Spell correct text using aspell\n            cleaned_text, spell_errors, markup_text = util_functions.spell_correct(self._text[len(self._text) - 1])\n            self._clean_text.append(cleaned_text)\n            self._spelling_errors.append(spell_errors)\n            self._markup_text.append(markup_text)\n            # Tokenize text\n            self._tokens.append(nltk.word_tokenize(self._clean_text[len(self._clean_text) - 1]))\n            # Part of speech tag text\n            self._pos.append(nltk.pos_tag(self._clean_text[len(self._clean_text) - 1].split(\" \")))\n            self._generated.append(essay_generated)\n            # Stem spell corrected text\n            porter = nltk.PorterStemmer()\n            por_toks = \" \".join([porter.stem(w) for w in self._tokens[len(self._tokens) - 1]])\n            self._clean_stem_text.append(por_toks)\n\n            ret = \"text: \" + self._text[len(self._text) - 1] + \" score: \" + str(essay_score)\n        else:\n            raise util_functions.InputError(essay_text, \"arguments need to be in format \"\n                                                        \"(text,score). text needs to be string,\"\n                                                        \" score needs to be int.\")"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef update_prompt(self, prompt_text):\n        if(isinstance(prompt_text, basestring)):\n            self._prompt = util_functions.sub_chars(prompt_text)\n            ret = self._prompt\n        else:\n            raise util_functions.InputError(prompt_text, \"Invalid prompt. Need to enter a string value.\")\n        return ret", "response": "Update the prompt string which is a string."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsubstitutes synonyms to generate extra essays from existing ones. This is done to increase the amount of training data. Should only be used with lowest scoring essays. e_text is the text of the original essay. e_score is the score of the original essay. dictionary is a fixed dictionary (list) of words to replace. max_syns defines the maximum number of additional essays to generate. Do not set too high.", "response": "def generate_additional_essays(self, e_text, e_score, dictionary=None, max_syns=3):\n        \"\"\"\n        Substitute synonyms to generate extra essays from existing ones.\n        This is done to increase the amount of training data.\n        Should only be used with lowest scoring essays.\n        e_text is the text of the original essay.\n        e_score is the score of the original essay.\n        dictionary is a fixed dictionary (list) of words to replace.\n        max_syns defines the maximum number of additional essays to generate.  Do not set too high.\n        \"\"\"\n        e_toks = nltk.word_tokenize(e_text)\n        all_syns = []\n        for word in e_toks:\n            synonyms = util_functions.get_wordnet_syns(word)\n            if(len(synonyms) > max_syns):\n                synonyms = random.sample(synonyms, max_syns)\n            all_syns.append(synonyms)\n        new_essays = []\n        for i in range(0, max_syns):\n            syn_toks = e_toks\n            for z in range(0, len(e_toks)):\n                if len(all_syns[z]) > i and (dictionary == None or e_toks[z] in dictionary):\n                    syn_toks[z] = all_syns[z][i]\n            new_essays.append(\" \".join(syn_toks))\n        for z in xrange(0, len(new_essays)):\n            self.add_essay(new_essays[z], e_score, 1)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a new code block from input text associated scores a prompt and a path to the model.", "response": "def create(text,score,prompt_string, dump_data=False):\n    \"\"\"\n    Creates a machine learning model from input text, associated scores, a prompt, and a path to the model\n    TODO: Remove model path argument, it is needed for now to support legacy code\n    text - A list of strings containing the text of the essays\n    score - a list of integers containing score values\n    prompt_string - the common prompt for the set of essays\n    \"\"\"\n\n    if dump_data:\n        dump_input_data(text, score)\n\n    algorithm = select_algorithm(score)\n    #Initialize a results dictionary to return\n    results = {'errors': [],'success' : False, 'cv_kappa' : 0, 'cv_mean_absolute_error': 0,\n               'feature_ext' : \"\", 'classifier' : \"\", 'algorithm' : algorithm,\n               'score' : score, 'text' : text, 'prompt' : prompt_string}\n\n    if len(text)!=len(score):\n        msg = \"Target and text lists must be same length.\"\n        results['errors'].append(msg)\n        log.exception(msg)\n        return results\n\n    try:\n        #Create an essay set object that encapsulates all the essays and alternate representations (tokens, etc)\n        e_set = model_creator.create_essay_set(text, score, prompt_string)\n    except:\n        msg = \"essay set creation failed.\"\n        results['errors'].append(msg)\n        log.exception(msg)\n    try:\n        #Gets features from the essay set and computes error\n        feature_ext, classifier, cv_error_results = model_creator.extract_features_and_generate_model(e_set, algorithm = algorithm)\n        results['cv_kappa']=cv_error_results['kappa']\n        results['cv_mean_absolute_error']=cv_error_results['mae']\n        results['feature_ext']=feature_ext\n        results['classifier']=classifier\n        results['algorithm'] = algorithm\n        results['success']=True\n    except:\n        msg = \"feature extraction and model creation failed.\"\n        results['errors'].append(msg)\n        log.exception(msg)\n\n    return results"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a generic list of numeric values and text values and a generic list of target variable.", "response": "def create_generic(numeric_values, textual_values, target, algorithm = util_functions.AlgorithmTypes.regression):\n    \"\"\"\n    Creates a model from a generic list numeric values and text values\n    numeric_values - A list of lists that are the predictors\n    textual_values - A list of lists that are the predictors\n    (each item in textual_values corresponds to the similarly indexed counterpart in numeric_values)\n    target - The variable that we are trying to predict.  A list of integers.\n    algorithm - the type of algorithm that will be used\n    \"\"\"\n\n    algorithm = select_algorithm(target)\n    #Initialize a result dictionary to return.\n    results = {'errors': [],'success' : False, 'cv_kappa' : 0, 'cv_mean_absolute_error': 0,\n               'feature_ext' : \"\", 'classifier' : \"\", 'algorithm' : algorithm}\n\n    if len(numeric_values)!=len(textual_values) or len(numeric_values)!=len(target):\n        msg = \"Target, numeric features, and text features must all be the same length.\"\n        results['errors'].append(msg)\n        log.exception(msg)\n        return results\n\n    try:\n        #Initialize a predictor set object that encapsulates all of the text and numeric predictors\n        pset = predictor_set.PredictorSet(essaytype=\"train\")\n        for i in xrange(0, len(numeric_values)):\n            pset.add_row(numeric_values[i], textual_values[i], target[i])\n    except:\n        msg = \"predictor set creation failed.\"\n        results['errors'].append(msg)\n        log.exception(msg)\n\n    try:\n        #Extract all features and then train a classifier with the features\n        feature_ext, classifier, cv_error_results = model_creator.extract_features_and_generate_model_predictors(pset, algorithm)\n        results['cv_kappa']=cv_error_results['kappa']\n        results['cv_mean_absolute_error']=cv_error_results['mae']\n        results['feature_ext']=feature_ext\n        results['classifier']=classifier\n        results['success']=True\n    except:\n        msg = \"feature extraction and model creation failed.\"\n        results['errors'].append(msg)\n        log.exception(msg)\n\n    return results"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates an essay set from given data.", "response": "def create_essay_set(text, score, prompt_string, generate_additional=True):\n    \"\"\"\n    Creates an essay set from given data.\n    Text should be a list of strings corresponding to essay text.\n    Score should be a list of scores where score[n] corresponds to text[n]\n    Prompt string is just a string containing the essay prompt.\n    Generate_additional indicates whether to generate additional essays at the minimum score point or not.\n    \"\"\"\n    x = EssaySet()\n    for i in xrange(0, len(text)):\n        x.add_essay(text[i], score[i])\n        if score[i] == min(score) and generate_additional == True:\n            x.generate_additional_essays(x._clean_text[len(x._clean_text) - 1], score[i])\n\n    x.update_prompt(prompt_string)\n\n    return x"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_cv_error(clf,feats,scores):\n    results={'success' : False, 'kappa' : 0, 'mae' : 0}\n    try:\n        cv_preds=util_functions.gen_cv_preds(clf,feats,scores)\n        err=numpy.mean(numpy.abs(numpy.array(cv_preds)-scores))\n        kappa=util_functions.quadratic_weighted_kappa(list(cv_preds),scores)\n        results['mae']=err\n        results['kappa']=kappa\n        results['success']=True\n    except ValueError as ex:\n        # If this is hit, everything is fine.  It is hard to explain why the error occurs, but it isn't a big deal.\n        msg = u\"Not enough classes (0,1,etc) in each cross validation fold: {ex}\".format(ex=ex)\n        log.debug(msg)\n    except:\n        log.exception(\"Error getting cv error estimates.\")\n\n    return results", "response": "Gets the cross validated error for a given classifier set of features and scores"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget two classifiers for each type of algorithm and returns them.", "response": "def get_algorithms(algorithm):\n    \"\"\"\n    Gets two classifiers for each type of algorithm, and returns them.  First for predicting, second for cv error.\n    type - one of util_functions.AlgorithmTypes\n    \"\"\"\n    if algorithm == util_functions.AlgorithmTypes.classification:\n        clf = sklearn.ensemble.GradientBoostingClassifier(n_estimators=100, learn_rate=.05,\n            max_depth=4, random_state=1,min_samples_leaf=3)\n        clf2=sklearn.ensemble.GradientBoostingClassifier(n_estimators=100, learn_rate=.05,\n            max_depth=4, random_state=1,min_samples_leaf=3)\n    else:\n        clf = sklearn.ensemble.GradientBoostingRegressor(n_estimators=100, learn_rate=.05,\n            max_depth=4, random_state=1,min_samples_leaf=3)\n        clf2=sklearn.ensemble.GradientBoostingRegressor(n_estimators=100, learn_rate=.05,\n            max_depth=4, random_state=1,min_samples_leaf=3)\n    return clf, clf2"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nextracts features and generates predictors based on a given predictor set.", "response": "def extract_features_and_generate_model_predictors(predictor_set, algorithm=util_functions.AlgorithmTypes.regression):\n    \"\"\"\n    Extracts features and generates predictors based on a given predictor set\n    predictor_set - a PredictorSet object that has been initialized with data\n    type - one of util_functions.AlgorithmType\n    \"\"\"\n    if(algorithm not in [util_functions.AlgorithmTypes.regression, util_functions.AlgorithmTypes.classification]):\n        algorithm = util_functions.AlgorithmTypes.regression\n\n    f = predictor_extractor.PredictorExtractor()\n    f.initialize_dictionaries(predictor_set)\n\n    train_feats = f.gen_feats(predictor_set)\n\n    clf,clf2 = get_algorithms(algorithm)\n    cv_error_results=get_cv_error(clf2,train_feats,predictor_set._target)\n\n    try:\n        set_score = numpy.asarray(predictor_set._target, dtype=numpy.int)\n        clf.fit(train_feats, set_score)\n    except ValueError:\n        log.exception(\"Not enough classes (0,1,etc) in sample.\")\n        set_score = predictor_set._target\n        set_score[0]=1\n        set_score[1]=0\n        clf.fit(train_feats, set_score)\n\n    return f, clf, cv_error_results"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nextracts features and generate a classifier based on the training set.", "response": "def extract_features_and_generate_model(essays, algorithm=util_functions.AlgorithmTypes.regression):\n    \"\"\"\n    Feed in an essay set to get feature vector and classifier\n    essays must be an essay set object\n    additional array is an optional argument that can specify\n    a numpy array of values to add in\n    returns a trained FeatureExtractor object and a trained classifier\n    \"\"\"\n    f = feature_extractor.FeatureExtractor()\n    f.initialize_dictionaries(essays)\n\n    train_feats = f.gen_feats(essays)\n\n    set_score = numpy.asarray(essays._score, dtype=numpy.int)\n    if len(util_functions.f7(list(set_score)))>5:\n        algorithm = util_functions.AlgorithmTypes.regression\n    else:\n        algorithm = util_functions.AlgorithmTypes.classification\n\n    clf,clf2 = get_algorithms(algorithm)\n\n    cv_error_results=get_cv_error(clf2,train_feats,essays._score)\n\n    try:\n        clf.fit(train_feats, set_score)\n    except ValueError:\n        log.exception(\"Not enough classes (0,1,etc) in sample.\")\n        set_score[0]=1\n        set_score[1]=0\n        clf.fit(train_feats, set_score)\n\n    return f, clf, cv_error_results"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef dump_model_to_file(prompt_string, feature_ext, classifier, text, score, model_path):\n    model_file = {'prompt': prompt_string, 'extractor': feature_ext, 'model': classifier, 'text' : text, 'score' : score}\n    pickle.dump(model_file, file=open(model_path, \"w\"))", "response": "Writes out a model to a file."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef create_essay_set_and_dump_model(text,score,prompt,model_path,additional_array=None):\n    essay_set=create_essay_set(text,score,prompt)\n    feature_ext,clf=extract_features_and_generate_model(essay_set,additional_array)\n    dump_model_to_file(prompt,feature_ext,clf,model_path)", "response": "Function that creates essay set extracts features and writes out model to file"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef initialize_dictionaries(self, p_set):\n        success = False\n        if not (hasattr(p_set, '_type')):\n            error_message = \"needs to be an essay set of the train type.\"\n            log.exception(error_message)\n            raise util_functions.InputError(p_set, error_message)\n\n        if not (p_set._type == \"train\"):\n            error_message = \"needs to be an essay set of the train type.\"\n            log.exception(error_message)\n            raise util_functions.InputError(p_set, error_message)\n\n        div_length=len(p_set._essay_sets)\n        if div_length==0:\n            div_length=1\n\n        #Ensures that even with a large amount of input textual features, training time stays reasonable\n        max_feats2 = int(math.floor(200/div_length))\n        for i in xrange(0,len(p_set._essay_sets)):\n            self._extractors.append(FeatureExtractor())\n            self._extractors[i].initialize_dictionaries(p_set._essay_sets[i], max_feats2=max_feats2)\n            self._initialized = True\n            success = True\n        return success", "response": "Initialize the dictionary with the textual inputs in the PredictorSet object that has been fed in in."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ngenerate features based on an iput p_set", "response": "def gen_feats(self, p_set):\n        \"\"\"\n        Generates features based on an iput p_set\n        p_set - PredictorSet\n        \"\"\"\n        if self._initialized!=True:\n            error_message = \"Dictionaries have not been initialized.\"\n            log.exception(error_message)\n            raise util_functions.InputError(p_set, error_message)\n\n        textual_features = []\n        for i in xrange(0,len(p_set._essay_sets)):\n            textual_features.append(self._extractors[i].gen_feats(p_set._essay_sets[i]))\n\n        textual_matrix = numpy.concatenate(textual_features, axis=1)\n        predictor_matrix = numpy.array(p_set._numeric_features)\n\n        print textual_matrix.shape\n        print predictor_matrix.shape\n\n        overall_matrix = numpy.concatenate((textual_matrix, predictor_matrix), axis=1)\n\n        return overall_matrix.copy()"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef t_text(self, t):\n        r':\\s*<text>'\n        t.lexer.text_start = t.lexer.lexpos - len('<text>')\n        t.lexer.begin('text')", "response": "r \\ s*text \\ s *"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef t_text_end(self, t):\n        r'</text>\\s*'\n        t.type = 'TEXT'\n        t.value = t.lexer.lexdata[\n            t.lexer.text_start:t.lexer.lexpos]\n        t.lexer.lineno += t.value.count('\\n')\n        t.value = t.value.strip()\n        t.lexer.begin('INITIAL')\n        return t", "response": "r End of text"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef t_KEYWORD_AS_TAG(self, t):\n        r'[a-zA-Z]+'\n        t.type = self.reserved.get(t.value, 'UNKNOWN_TAG')\n        t.value = t.value.strip()\n        return t", "response": "r Keyword as tag"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef t_LINE_OR_KEYWORD_VALUE(self, t):\n        r':.+'\n        t.value = t.value[1:].strip()\n        if t.value in self.reserved.keys():\n            t.type = self.reserved[t.value]\n        else:\n            t.type = 'LINE'\n        return t", "response": "t is a KeywordType instance"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef tv_to_rdf(infile_name, outfile_name):\n    parser = Parser(Builder(), StandardLogger())\n    parser.build()\n    with open(infile_name) as infile:\n        data = infile.read()\n        document, error = parser.parse(data)\n        if not error:\n            with open(outfile_name, mode='w') as outfile:\n                write_document(document, outfile)\n            return True\n        else:\n            print('Errors encountered while parsing RDF file.')\n            messages = []\n            document.validate(messages)\n            print('\\n'.join(messages))\n            return False", "response": "Convert a SPDX file from tag value format to RDF format."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef order_error(self, first_tag, second_tag, line):\n        self.error = True\n        msg = ERROR_MESSAGES['A_BEFORE_B'].format(first_tag, second_tag, line)\n        self.logger.log(msg)", "response": "Reports an OrderError. Error message will state that\n        first_tag came before second_tag."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef p_lic_xref_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.add_lic_xref(self.document, value)\n        except OrderError:\n            self.order_error('LicenseCrossReference', 'LicenseName', p.lineno(1))", "response": "\\ x1b [ 1mNAME \\ x1b [ 0m lic_xref - Adds a LICS_CRS_REF LINE to the document."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef p_extr_lic_name_1(self, p):\n        try:\n            self.builder.set_lic_name(self.document, p[2])\n        except OrderError:\n            self.order_error('LicenseName', 'LicenseID', p.lineno(1))\n        except CardinalityError:\n            self.more_than_one_error('LicenseName', p.lineno(1))", "response": "Parse the LICS_NAME section."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef p_prj_home_art_1(self, p):\n        try:\n            self.builder.set_file_atrificat_of_project(self.document, 'home', p[2])\n        except OrderError:\n            self.order_error('ArtificatOfProjectHomePage', 'FileName', p.lineno(1))", "response": "prj_home_art : ART_PRJ_HOME LINE"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef p_file_notice_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.set_file_notice(self.document, value)\n        except OrderError:\n            self.order_error('FileNotice', 'FileName', p.lineno(1))\n        except CardinalityError:\n            self.more_than_one_error('FileNotice', p.lineno(1))", "response": "\\ x1b [ 1mNAME \\ x1b [ 0m file_notice - Set the file_notice field to the value of the file_notice field."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef p_file_cr_text_1(self, p):\n        try:\n            self.builder.set_file_copyright(self.document, p[2])\n        except OrderError:\n            self.order_error('FileCopyrightText', 'FileName', p.lineno(1))\n        except CardinalityError:\n            self.more_than_one_error('FileCopyrightText', p.lineno(1))", "response": "file_cr_text : FILE_CR_TEXT file_cr_value"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef p_file_lics_info_1(self, p):\n        try:\n            self.builder.set_file_license_in_file(self.document, p[2])\n        except OrderError:\n            self.order_error('LicenseInfoInFile', 'FileName', p.lineno(1))\n        except SPDXValueError:\n            self.error = True\n            msg = ERROR_MESSAGES['FILE_LICS_INFO_VALUE'].format(p.lineno(1))\n            self.logger.log(msg)", "response": "File License Info In File Method"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef p_file_name_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.set_file_name(self.document, value)\n        except OrderError:\n            self.order_error('FileName', 'PackageName', p.lineno(1))", "response": "\\ x1b [ 1mNAME \\ x1b [ 0m file_name - Set the file name of the user."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef p_file_comment_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.set_file_comment(self.document, value)\n        except OrderError:\n            self.order_error('FileComment', 'FileName', p.lineno(1))\n        except CardinalityError:\n            self.more_than_one_error('FileComment', p.lineno(1))", "response": "\\ x1b [ 1mNAME \\ x1b [ 0m file_comment - Set the file comment."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef p_file_type_1(self, p):\n        try:\n            self.builder.set_file_type(self.document, p[2])\n        except OrderError:\n            self.order_error('FileType', 'FileName', p.lineno(1))\n        except CardinalityError:\n            self.more_than_one_error('FileType', p.lineno(1))", "response": "\\ x1b [ 1mNAME \\ x1b [ 0m file_type - Set the file type of the class."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef p_file_conc_1(self, p):\n        try:\n            self.builder.set_concluded_license(self.document, p[2])\n        except SPDXValueError:\n            self.error = True\n            msg = ERROR_MESSAGES['FILE_LICS_CONC_VALUE'].format(p.lineno(1))\n            self.logger.log(msg)\n        except OrderError:\n            self.order_error('LicenseConcluded', 'FileName', p.lineno(1))\n        except CardinalityError:\n            self.more_than_one_error('LicenseConcluded', p.lineno(1))", "response": "File Concretization Entry point."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef p_file_type_value(self, p):\n        if six.PY2:\n            p[0] = p[1].decode(encoding='utf-8')\n        else:\n            p[0] = p[1]", "response": "\\ x1b [ 1mNAME \\ x1b [ 0m file_type_value - Gets the file type value of the log entry."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef p_pkg_desc_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.set_pkg_desc(self.document, value)\n        except CardinalityError:\n            self.more_than_one_error('PackageDescription', p.lineno(1))\n        except OrderError:\n            self.order_error('PackageDescription', 'PackageFileName', p.lineno(1))", "response": "pkg_desc : PKG_DESC TEXT"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef p_pkg_summary_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.set_pkg_summary(self.document, value)\n        except OrderError:\n            self.order_error('PackageSummary', 'PackageFileName', p.lineno(1))\n        except CardinalityError:\n            self.more_than_one_error('PackageSummary', p.lineno(1))", "response": "pkg_summary : PKG_SUM TEXT"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef p_pkg_cr_text_1(self, p):\n        try:\n            self.builder.set_pkg_cr_text(self.document, p[2])\n        except OrderError:\n            self.order_error('PackageCopyrightText', 'PackageFileName', p.lineno(1))\n        except CardinalityError:\n            self.more_than_one_error('PackageCopyrightText', p.lineno(1))", "response": "PackageCopyrightText | PackageFileName | PackageCopyrightText| | |"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef p_pkg_lic_ff_1(self, p):\n        try:\n            self.builder.set_pkg_license_from_file(self.document, p[2])\n        except OrderError:\n            self.order_error('PackageLicenseInfoFromFiles', 'PackageName', p.lineno(1))\n        except SPDXValueError:\n            self.error = True\n            msg = ERROR_MESSAGES['PKG_LIC_FFILE_VALUE'].format(p.lineno(1))\n            self.logger.log(msg)", "response": "Package LicenseInfoFromFiles entry point."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef p_pkg_lic_ff_value_3(self, p):\n        if six.PY2:\n            value = p[1].decode(encoding='utf-8')\n        else:\n            value = p[1]\n        p[0] = document.License.from_identifier(value)", "response": "pkg_lic_ff_value is a line of code that is used in the pkg_lic_ff_value_2 function."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef p_pkg_chksum_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.set_pkg_chk_sum(self.document, value)\n        except OrderError:\n            self.order_error('PackageChecksum', 'PackageFileName', p.lineno(1))\n        except CardinalityError:\n            self.more_than_one_error('PackageChecksum', p.lineno(1))", "response": "PackageChecksum | PKG_CHKSUM | PackageFileName | |"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef p_pkg_home_2(self, p):\n        self.error = True\n        msg = ERROR_MESSAGES['PKG_HOME_VALUE']\n        self.logger.log(msg)", "response": "pkg_home : PKG_HOME error"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef p_package_version_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.set_pkg_vers(self.document, value)\n        except OrderError:\n            self.order_error('PackageVersion', 'PackageName', p.lineno(1))\n        except CardinalityError:\n            self.more_than_one_error('PackageVersion', p.lineno(1))", "response": "package_version : PKG_VERSION LINE"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef p_package_name(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.create_package(self.document, value)\n        except CardinalityError:\n            self.more_than_one_error('PackageName', p.lineno(1))", "response": "\\ x1b [ 1mNAME \\ x1b [ 0m package_name - Creates a new package."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef p_annotation_date_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.add_annotation_date(self.document, value)\n        except CardinalityError:\n            self.more_than_one_error('AnnotationDate', p.lineno(1))\n        except OrderError:\n            self.order_error('AnnotationDate', 'Annotator', p.lineno(1))", "response": "Annotation date | Annotation date | Date |"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef p_annotation_comment_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.add_annotation_comment(self.document, value)\n        except CardinalityError:\n            self.more_than_one_error('AnnotationComment', p.lineno(1))\n        except OrderError:\n            self.order_error('AnnotationComment', 'Annotator', p.lineno(1))", "response": "annotation_comment : ANNOTATION_COMMENT TEXT"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef p_ext_doc_refs_1(self, p):\n        try:\n            if six.PY2:\n                doc_ref_id = p[2].decode(encoding='utf-8')\n                doc_uri = p[3].decode(encoding='utf-8')\n                ext_doc_chksum = p[4].decode(encoding='utf-8')\n            else:\n                doc_ref_id = p[2]\n                doc_uri = p[3]\n                ext_doc_chksum = p[4]\n\n            self.builder.add_ext_doc_refs(self.document, doc_ref_id, doc_uri,\n                                          ext_doc_chksum)\n        except SPDXValueError:\n            self.error = True\n            msg = ERROR_MESSAGES['EXT_DOC_REF_VALUE'].format(p.lineno(2))\n            self.logger.log(msg)", "response": "\\ x1b [ 1mNAME \\ x1b [ 0m ext_doc_refs - Adds a new entry to the list of external doc_refs."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef p_creator_comment_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.set_creation_comment(self.document, value)\n        except CardinalityError:\n            self.more_than_one_error('CreatorComment', p.lineno(1))", "response": "\\ x1b [ 1mNAME \\ x1b [ 0m creator_comment - Creates a CREATOR_COMMENT."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef p_created_1(self, p):\n        try:\n            if six.PY2:\n                value = p[2].decode(encoding='utf-8')\n            else:\n                value = p[2]\n            self.builder.set_created_date(self.document, value)\n        except CardinalityError:\n            self.more_than_one_error('Created', p.lineno(1))", "response": "\\ x1b [ 1mNAME \\ x1b [ 0m created - Gets the date of the created item."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting the creation info to out.", "response": "def write_creation_info(creation_info, out):\n    \"\"\"\n    Write the creation info to out.\n    \"\"\"\n    out.write('# Creation Info\\n\\n')\n    # Write sorted creators\n    for creator in sorted(creation_info.creators):\n        write_value('Creator', creator, out)\n\n    # write created\n    write_value('Created', creation_info.created_iso_format, out)\n    # possible comment\n    if creation_info.has_comment:\n        write_text_value('CreatorComment', creation_info.comment, out)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef write_review(review, out):\n    out.write('# Review\\n\\n')\n    write_value('Reviewer', review.reviewer, out)\n    write_value('ReviewDate', review.review_date_iso_format, out)\n    if review.has_comment:\n        write_text_value('ReviewComment', review.comment, out)", "response": "Write the fields of a single review to out."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef write_annotation(annotation, out):\n    out.write('# Annotation\\n\\n')\n    write_value('Annotator', annotation.annotator, out)\n    write_value('AnnotationDate', annotation.annotation_date_iso_format, out)\n    if annotation.has_comment:\n        write_text_value('AnnotationComment', annotation.comment, out)\n    write_value('AnnotationType', annotation.annotation_type, out)\n    write_value('SPDXREF', annotation.spdx_id, out)", "response": "Writes the fields of an annotation to out."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nwriting a single file to out.", "response": "def write_file(spdx_file, out):\n    \"\"\"\n    Write a file fields to out.\n    \"\"\"\n    out.write('# File\\n\\n')\n    write_value('FileName', spdx_file.name, out)\n    write_value('SPDXID', spdx_file.spdx_id, out)\n    if spdx_file.has_optional_field('type'):\n        write_file_type(spdx_file.type, out)\n    write_value('FileChecksum', spdx_file.chk_sum.to_tv(), out)\n    if isinstance(spdx_file.conc_lics, (document.LicenseConjunction, document.LicenseDisjunction)):\n        write_value('LicenseConcluded', u'({0})'.format(spdx_file.conc_lics), out)\n    else:\n        write_value('LicenseConcluded', spdx_file.conc_lics, out)\n\n    # write sorted list\n    for lics in sorted(spdx_file.licenses_in_file):\n        write_value('LicenseInfoInFile', lics, out)\n\n    if isinstance(spdx_file.copyright, six.string_types):\n        write_text_value('FileCopyrightText', spdx_file.copyright, out)\n    else:\n        write_value('FileCopyrightText', spdx_file.copyright, out)\n\n    if spdx_file.has_optional_field('license_comment'):\n        write_text_value('LicenseComments', spdx_file.license_comment, out)\n\n    if spdx_file.has_optional_field('comment'):\n        write_text_value('FileComment', spdx_file.comment, out)\n\n    if spdx_file.has_optional_field('notice'):\n        write_text_value('FileNotice', spdx_file.notice, out)\n\n    for contributor in sorted(spdx_file.contributors):\n        write_value('FileContributor', contributor, out)\n\n    for dependency in sorted(spdx_file.dependencies):\n        write_value('FileDependency', dependency, out)\n\n    names = spdx_file.artifact_of_project_name\n    homepages = spdx_file.artifact_of_project_home\n    uris = spdx_file.artifact_of_project_uri\n\n    for name, homepage, uri in sorted(zip_longest(names, homepages, uris)):\n        write_value('ArtifactOfProjectName', name, out)\n        if homepage is not None:\n            write_value('ArtifactOfProjectHomePage', homepage, out)\n        if uri is not None:\n            write_value('ArtifactOfProjectURI', uri, out)"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef write_package(package, out):\n    out.write('# Package\\n\\n')\n    write_value('PackageName', package.name, out)\n    if package.has_optional_field('version'):\n        write_value('PackageVersion', package.version, out)\n    write_value('PackageDownloadLocation', package.download_location, out)\n\n    if package.has_optional_field('summary'):\n        write_text_value('PackageSummary', package.summary, out)\n\n    if package.has_optional_field('source_info'):\n        write_text_value('PackageSourceInfo', package.source_info, out)\n\n    if package.has_optional_field('file_name'):\n        write_value('PackageFileName', package.file_name, out)\n\n    if package.has_optional_field('supplier'):\n        write_value('PackageSupplier', package.supplier, out)\n\n    if package.has_optional_field('originator'):\n        write_value('PackageOriginator', package.originator, out)\n\n    if package.has_optional_field('check_sum'):\n        write_value('PackageChecksum', package.check_sum.to_tv(), out)\n\n    write_value('PackageVerificationCode', format_verif_code(package), out)\n\n    if package.has_optional_field('description'):\n        write_text_value('PackageDescription', package.description, out)\n\n    if isinstance(package.license_declared, (document.LicenseConjunction,\n        document.LicenseDisjunction)):\n        write_value('PackageLicenseDeclared', u'({0})'.format(package.license_declared), out)\n    else:\n        write_value('PackageLicenseDeclared', package.license_declared, out)\n\n    if isinstance(package.conc_lics, (document.LicenseConjunction,\n        document.LicenseDisjunction)):\n        write_value('PackageLicenseConcluded', u'({0})'.format(package.conc_lics), out)\n    else:\n        write_value('PackageLicenseConcluded', package.conc_lics, out)\n\n    # Write sorted list of licenses.\n    for lics in sorted(package.licenses_from_files):\n        write_value('PackageLicenseInfoFromFiles', lics, out)\n\n    if package.has_optional_field('license_comment'):\n        write_text_value('PackageLicenseComments', package.license_comment, out)\n\n    # cr_text is either free form text or NONE or NOASSERTION.\n    if isinstance(package.cr_text, six.string_types):\n        write_text_value('PackageCopyrightText', package.cr_text, out)\n    else:\n        write_value('PackageCopyrightText', package.cr_text, out)\n\n    if package.has_optional_field('homepage'):\n        write_value('PackageHomePage', package.homepage, out)\n\n    # Write sorted files.\n    for spdx_file in sorted(package.files):\n        write_separators(out)\n        write_file(spdx_file, out)", "response": "Write a single package to out."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef write_extracted_licenses(lics, out):\n    write_value('LicenseID', lics.identifier, out)\n\n    if lics.full_name is not None:\n        write_value('LicenseName', lics.full_name, out)\n\n    if lics.comment is not None:\n        write_text_value('LicenseComment', lics.comment, out)\n\n    for xref in sorted(lics.cross_ref):\n        write_value('LicenseCrossReference', xref, out)\n\n    write_text_value('ExtractedText', lics.text, out)", "response": "Write extracted licenses to out."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nwrites an SPDX document to out.", "response": "def write_document(document, out, validate=True):\n    \"\"\"\n    Write an SPDX tag value document.\n    - document - spdx.document instance.\n    - out - file like object that will be written to.\n    Optionally `validate` the document before writing and raise\n    InvalidDocumentError if document.validate returns False.\n    \"\"\"\n    messages = []\n    messages = document.validate(messages)\n    if validate and messages:\n        raise InvalidDocumentError(messages)\n\n    # Write out document information\n    out.write('# Document Information\\n\\n')\n    write_value('SPDXVersion', str(document.version), out)\n    write_value('DataLicense', document.data_license.identifier, out)\n    write_value('DocumentName', document.name, out)\n    write_value('SPDXID', 'SPDXRef-DOCUMENT', out)\n    write_value('DocumentNamespace', document.namespace, out)\n    if document.has_comment:\n        write_text_value('DocumentComment', document.comment, out)\n    for doc_ref in document.ext_document_references:\n        doc_ref_str = ' '.join([doc_ref.external_document_id,\n                                doc_ref.spdx_document_uri,\n                                doc_ref.check_sum.identifier + ':' +\n                                doc_ref.check_sum.value])\n        write_value('ExternalDocumentRef', doc_ref_str, out)\n    write_separators(out)\n    # Write out creation info\n    write_creation_info(document.creation_info, out)\n    write_separators(out)\n\n    # Writesorted reviews\n    for review in sorted(document.reviews):\n        write_review(review, out)\n        write_separators(out)\n\n    #Write sorted annotations\n    for annotation in sorted(document.annotations):\n        write_annotation(annotation, out)\n        write_separators(out)\n\n    # Write out package info\n    write_package(document.package, out)\n    write_separators(out)\n\n    out.write('# Extracted Licenses\\n\\n')\n    for lic in sorted(document.extracted_licenses):\n        write_extracted_licenses(lic, out)\n        write_separators(out)"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef checksum_from_sha1(value):\n    # More constrained regex at lexer level\n    CHECKSUM_RE = re.compile('SHA1:\\s*([\\S]+)', re.UNICODE)\n    match = CHECKSUM_RE.match(value)\n    if match:\n        return checksum.Algorithm(identifier='SHA1', value=match.group(1))\n    else:\n        return None", "response": "Returns an spdx. checksum. Algorithm instance representing the SHA1\n    checksum or None if the value does not match the SHA1 checksum."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef str_from_text(text):\n    REGEX = re.compile('<text>((.|\\n)+)</text>', re.UNICODE)\n    match = REGEX.match(text)\n    if match:\n        return match.group(1)\n    else:\n        return None", "response": "Return content of a free form text block as a string."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_doc_version(self, doc, value):\n        if not self.doc_version_set:\n            self.doc_version_set = True\n            m = self.VERS_STR_REGEX.match(value)\n            if m is None:\n                raise SPDXValueError('Document::Version')\n            else:\n                doc.version = version.Version(major=int(m.group(1)),\n                                              minor=int(m.group(2)))\n                return True\n        else:\n            raise CardinalityError('Document::Version')", "response": "Set the document version."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef set_doc_data_lics(self, doc, lics):\n        if not self.doc_data_lics_set:\n            self.doc_data_lics_set = True\n            if validations.validate_data_lics(lics):\n                doc.data_license = document.License.from_identifier(lics)\n                return True\n            else:\n                raise SPDXValueError('Document::DataLicense')\n        else:\n            raise CardinalityError('Document::DataLicense')", "response": "Sets the document data license. Raises value error if malformed value CardinalityError\n            Raises value error if malformed value CardinalityError\n            raises CardinalityError\n           "}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsets the document name.", "response": "def set_doc_name(self, doc, name):\n        \"\"\"Sets the document name.\n        Raises CardinalityError if already defined.\n        \"\"\"\n        if not self.doc_name_set:\n            doc.name = name\n            self.doc_name_set = True\n            return True\n        else:\n            raise CardinalityError('Document::Name')"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_doc_spdx_id(self, doc, doc_spdx_id_line):\n        if not self.doc_spdx_id_set:\n            if doc_spdx_id_line == 'SPDXRef-DOCUMENT':\n                doc.spdx_id = doc_spdx_id_line\n                self.doc_spdx_id_set = True\n                return True\n            else:\n                raise SPDXValueError('Document::SPDXID')\n        else:\n            raise CardinalityError('Document::SPDXID')", "response": "Sets the document SPDX Identifier. Raises value error if malformed value CardinalityError if CardinalityError is raised."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef reset_document(self):\n        # FIXME: this state does not make sense\n        self.doc_version_set = False\n        self.doc_comment_set = False\n        self.doc_namespace_set = False\n        self.doc_data_lics_set = False\n        self.doc_name_set = False\n        self.doc_spdx_id_set = False", "response": "Resets the state to allow building new documents"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nset the spdx_document_uri attribute of the ExternalDocumentRef object.", "response": "def set_spdx_doc_uri(self, doc, spdx_doc_uri):\n        \"\"\"\n        Sets the `spdx_document_uri` attribute of the `ExternalDocumentRef`\n        object.\n        \"\"\"\n        if validations.validate_doc_namespace(spdx_doc_uri):\n            doc.ext_document_references[-1].spdx_document_uri = spdx_doc_uri\n        else:\n            raise SPDXValueError('Document::ExternalDocumentRef')"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nbuild a tool object out of a string representation.", "response": "def build_tool(self, doc, entity):\n        \"\"\"Builds a tool object out of a string representation.\n        Returns built tool. Raises SPDXValueError if failed to extract\n        tool name or name is malformed\n        \"\"\"\n        match = self.tool_re.match(entity)\n        if match and validations.validate_tool_name(match.group(self.TOOL_NAME_GROUP)):\n            name = match.group(self.TOOL_NAME_GROUP)\n            return creationinfo.Tool(name)\n        else:\n            raise SPDXValueError('Failed to extract tool name')"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef build_org(self, doc, entity):\n        match = self.org_re.match(entity)\n        if match and validations.validate_org_name(match.group(self.ORG_NAME_GROUP)):\n            name = match.group(self.ORG_NAME_GROUP).strip()\n            email = match.group(self.ORG_EMAIL_GROUP)\n            if (email is not None) and (len(email) != 0):\n                return creationinfo.Organization(name=name, email=email.strip())\n            else:\n                return creationinfo.Organization(name=name, email=None)\n        else:\n            raise SPDXValueError('Failed to extract Organization name')", "response": "Builds an organization object of a string representation."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef build_person(self, doc, entity):\n        match = self.person_re.match(entity)\n        if match and validations.validate_person_name(match.group(self.PERSON_NAME_GROUP)):\n            name = match.group(self.PERSON_NAME_GROUP).strip()\n            email = match.group(self.PERSON_EMAIL_GROUP)\n            if (email is not None) and (len(email) != 0):\n                return creationinfo.Person(name=name, email=email.strip())\n            else:\n                return creationinfo.Person(name=name, email=None)\n        else:\n            raise SPDXValueError('Failed to extract person name')", "response": "Builds an organization object of a string representation."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add_creator(self, doc, creator):\n        if validations.validate_creator(creator):\n            doc.creation_info.add_creator(creator)\n            return True\n        else:\n            raise SPDXValueError('CreationInfo::Creator')", "response": "Adds a creator to the document s creation info. Returns True if creator is valid."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets the created date of the resource.", "response": "def set_created_date(self, doc, created):\n        \"\"\"Sets created date, Raises CardinalityError if\n        created date already set.\n        Raises SPDXValueError if created is not a date.\n        \"\"\"\n        if not self.created_date_set:\n            self.created_date_set = True\n            date = utils.datetime_from_iso_format(created)\n            if date is not None:\n                doc.creation_info.created = date\n                return True\n            else:\n                raise SPDXValueError('CreationInfo::Date')\n        else:\n            raise CardinalityError('CreationInfo::Created')"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting creation comment for the current object.", "response": "def set_creation_comment(self, doc, comment):\n        \"\"\"Sets creation comment, Raises CardinalityError if\n        comment already set.\n        Raises SPDXValueError if not free form text.\n        \"\"\"\n        if not self.creation_comment_set:\n            self.creation_comment_set = True\n            if validations.validate_creation_comment(comment):\n                doc.creation_info.comment = str_from_text(comment)\n                return True\n            else:\n                raise SPDXValueError('CreationInfo::Comment')\n        else:\n            raise CardinalityError('CreationInfo::Comment')"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the license list version of the user. Raises CardinalityError if the attribute license_list_ver is already set.", "response": "def set_lics_list_ver(self, doc, value):\n        \"\"\"Sets the license list version, Raises CardinalityError if\n        already set, SPDXValueError if incorrect value.\n        \"\"\"\n        if not self.lics_list_ver_set:\n            self.lics_list_ver_set = True\n            vers = version.Version.from_str(value)\n            if vers is not None:\n                doc.creation_info.license_list_version = vers\n                return True\n            else:\n                raise SPDXValueError('CreationInfo::LicenseListVersion')\n        else:\n            raise CardinalityError('CreationInfo::LicenseListVersion')"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nresets the builder state to allow building new creation info.", "response": "def reset_creation_info(self):\n        \"\"\"\n        Resets builder state to allow building new creation info.\"\"\"\n        # FIXME: this state does not make sense\n        self.created_date_set = False\n        self.creation_comment_set = False\n        self.lics_list_ver_set = False"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef add_reviewer(self, doc, reviewer):\n        # Each reviewer marks the start of a new review object.\n        # FIXME: this state does not make sense\n        self.reset_reviews()\n        if validations.validate_reviewer(reviewer):\n            doc.add_review(review.Review(reviewer=reviewer))\n            return True\n        else:\n            raise SPDXValueError('Review::Reviewer')", "response": "Adds a reviewer to the SPDX Document."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef add_review_comment(self, doc, comment):\n        if len(doc.reviews) != 0:\n            if not self.review_comment_set:\n                self.review_comment_set = True\n                if validations.validate_review_comment(comment):\n                    doc.reviews[-1].comment = str_from_text(comment)\n                    return True\n                else:\n                    raise SPDXValueError('ReviewComment::Comment')\n            else:\n                raise CardinalityError('ReviewComment')\n        else:\n            raise OrderError('ReviewComment')", "response": "Adds a review comment to the article. Raises CardinalityError if no reviewer defined before. Raises SPDXValueError if comment is not free form text. Raises OrderError if no reviewer defined before. Raises SPDXValueError if comment is not free form text. Returns True if the comment was valid."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nresets the builder s state to allow building new annotations.", "response": "def reset_annotations(self):\n        \"\"\"Resets the builder's state to allow building new annotations.\"\"\"\n        # FIXME: this state does not make sense\n        self.annotation_date_set = False\n        self.annotation_comment_set = False\n        self.annotation_type_set = False\n        self.annotation_spdx_id_set = False"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add_annotator(self, doc, annotator):\n        # Each annotator marks the start of a new annotation object.\n        # FIXME: this state does not make sense\n        self.reset_annotations()\n        if validations.validate_annotator(annotator):\n            doc.add_annotation(annotation.Annotation(annotator=annotator))\n            return True\n        else:\n            raise SPDXValueError('Annotation::Annotator')", "response": "Adds an annotator to the SPDX Document."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef add_annotation_date(self, doc, annotation_date):\n        if len(doc.annotations) != 0:\n            if not self.annotation_date_set:\n                self.annotation_date_set = True\n                date = utils.datetime_from_iso_format(annotation_date)\n                if date is not None:\n                    doc.annotations[-1].annotation_date = date\n                    return True\n                else:\n                    raise SPDXValueError('Annotation::AnnotationDate')\n            else:\n                raise CardinalityError('Annotation::AnnotationDate')\n        else:\n            raise OrderError('Annotation::AnnotationDate')", "response": "Adds an annotation date to the document. Raises CardinalityError if no annotator defined before. Raises SPDXValueError if invalid value. Raises OrderError if no annotator defined before. Raises SPDXValueError if invalid value. Returns True if the document has annotations and an annotation date is not set."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nadding an annotation comment to the document. Raises CardinalityError if no annotator defined before. Raises SPDXValueError if comment is not free form text. Raises OrderError if no annotator defined before. Raises SPDXValueError if comment is not free form text. Returns True if the comment was valid.", "response": "def add_annotation_comment(self, doc, comment):\n        \"\"\"Sets the annotation comment. Raises CardinalityError if\n        already set. OrderError if no annotator defined before.\n        Raises SPDXValueError if comment is not free form text.\n        \"\"\"\n        if len(doc.annotations) != 0:\n            if not self.annotation_comment_set:\n                self.annotation_comment_set = True\n                if validations.validate_annotation_comment(comment):\n                    doc.annotations[-1].comment = str_from_text(comment)\n                    return True\n                else:\n                    raise SPDXValueError('AnnotationComment::Comment')\n            else:\n                raise CardinalityError('AnnotationComment::Comment')\n        else:\n            raise OrderError('AnnotationComment::Comment')"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef add_annotation_type(self, doc, annotation_type):\n        if len(doc.annotations) != 0:\n            if not self.annotation_type_set:\n                self.annotation_type_set = True\n                if validations.validate_annotation_type(annotation_type):\n                    doc.annotations[-1].annotation_type = annotation_type\n                    return True\n                else:\n                    raise SPDXValueError('Annotation::AnnotationType')\n            else:\n                raise CardinalityError('Annotation::AnnotationType')\n        else:\n            raise OrderError('Annotation::AnnotationType')", "response": "Adds an annotation type to the annotation type list. Raises CardinalityError if no such annotation type is defined before. Raises SPDXValueError if invalid value."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef reset_package(self):\n        # FIXME: this state does not make sense\n        self.package_set = False\n        self.package_vers_set = False\n        self.package_file_name_set = False\n        self.package_supplier_set = False\n        self.package_originator_set = False\n        self.package_down_location_set = False\n        self.package_home_set = False\n        self.package_verif_set = False\n        self.package_chk_sum_set = False\n        self.package_source_info_set = False\n        self.package_conc_lics_set = False\n        self.package_license_declared_set = False\n        self.package_license_comment_set = False\n        self.package_cr_text_set = False\n        self.package_summary_set = False\n        self.package_desc_set = False", "response": "Resets the state of the internal state to build new packages."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef create_package(self, doc, name):\n        if not self.package_set:\n            self.package_set = True\n            doc.package = package.Package(name=name)\n            return True\n        else:\n            raise CardinalityError('Package::Name')", "response": "Creates a package for the SPDX Document."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset the version of the package in the given doc.", "response": "def set_pkg_vers(self, doc, version):\n        \"\"\"Sets package version, if not already set.\n        version - Any string.\n        Raises CardinalityError if already has a version.\n        Raises OrderError if no package previously defined.\n        \"\"\"\n        self.assert_package_exists()\n        if not self.package_vers_set:\n            self.package_vers_set = True\n            doc.package.version = version\n            return True\n        else:\n            raise CardinalityError('Package::Version')"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_pkg_file_name(self, doc, name):\n        self.assert_package_exists()\n        if not self.package_file_name_set:\n            self.package_file_name_set = True\n            doc.package.file_name = name\n            return True\n        else:\n            raise CardinalityError('Package::FileName')", "response": "Sets the package file name."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset the package supplier if not already set.", "response": "def set_pkg_supplier(self, doc, entity):\n        \"\"\"Sets the package supplier, if not already set.\n        entity - Organization, Person or NoAssert.\n        Raises CardinalityError if already has a supplier.\n        Raises OrderError if no package previously defined.\n        \"\"\"\n        self.assert_package_exists()\n        if not self.package_supplier_set:\n            self.package_supplier_set = True\n            if validations.validate_pkg_supplier(entity):\n                doc.package.supplier = entity\n                return True\n            else:\n                raise SPDXValueError('Package::Supplier')\n        else:\n            raise CardinalityError('Package::Supplier')"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the package originator if not already set.", "response": "def set_pkg_originator(self, doc, entity):\n        \"\"\"Sets the package originator, if not already set.\n        entity - Organization, Person or NoAssert.\n        Raises CardinalityError if already has an originator.\n        Raises OrderError if no package previously defined.\n        \"\"\"\n        self.assert_package_exists()\n        if not self.package_originator_set:\n            self.package_originator_set = True\n            if validations.validate_pkg_originator(entity):\n                doc.package.originator = entity\n                return True\n            else:\n                raise SPDXValueError('Package::Originator')\n        else:\n            raise CardinalityError('Package::Originator')"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset the package download location.", "response": "def set_pkg_down_location(self, doc, location):\n        \"\"\"Sets the package download location, if not already set.\n        location - A string\n        Raises CardinalityError if already defined.\n        Raises OrderError if no package previously defined.\n        \"\"\"\n        self.assert_package_exists()\n        if not self.package_down_location_set:\n            self.package_down_location_set = True\n            doc.package.download_location = location\n            return True\n        else:\n            raise CardinalityError('Package::DownloadLocation')"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsets the package homepage location if not already set.", "response": "def set_pkg_home(self, doc, location):\n        \"\"\"Sets the package homepage location if not already set.\n        location - A string or None or NoAssert.\n        Raises CardinalityError if already defined.\n        Raises OrderError if no package previously defined.\n        Raises SPDXValueError if location has incorrect value.\n        \"\"\"\n        self.assert_package_exists()\n        if not self.package_home_set:\n            self.package_home_set = True\n            if validations.validate_pkg_homepage(location):\n                doc.package.homepage = location\n                return True\n            else:\n                raise SPDXValueError('Package::HomePage')\n        else:\n            raise CardinalityError('Package::HomePage')"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting the verification code for the given package.", "response": "def set_pkg_verif_code(self, doc, code):\n        \"\"\"Sets the package verification code, if not already set.\n        code - A string.\n        Raises CardinalityError if already defined.\n        Raises OrderError if no package previously defined.\n        Raises Value error if doesn't match verifcode form\n        \"\"\"\n        self.assert_package_exists()\n        if not self.package_verif_set:\n            self.package_verif_set = True\n            match = self.VERIF_CODE_REGEX.match(code)\n            if match:\n                doc.package.verif_code = match.group(self.VERIF_CODE_CODE_GRP)\n                if match.group(self.VERIF_CODE_EXC_FILES_GRP) is not None:\n                    doc.package.verif_exc_files = match.group(self.VERIF_CODE_EXC_FILES_GRP).split(',')\n                return True\n            else:\n                raise SPDXValueError('Package::VerificationCode')\n        else:\n            raise CardinalityError('Package::VerificationCode')"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef set_pkg_chk_sum(self, doc, chk_sum):\n        self.assert_package_exists()\n        if not self.package_chk_sum_set:\n            self.package_chk_sum_set = True\n            doc.package.check_sum = checksum_from_sha1(chk_sum)\n            return True\n        else:\n            raise CardinalityError('Package::CheckSum')", "response": "Sets the package check sum if not already set."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_pkg_source_info(self, doc, text):\n        self.assert_package_exists()\n        if not self.package_source_info_set:\n            self.package_source_info_set = True\n            if validations.validate_pkg_src_info(text):\n                doc.package.source_info = str_from_text(text)\n                return True\n            else:\n                raise SPDXValueError('Pacckage::SourceInfo')\n        else:\n            raise CardinalityError('Package::SourceInfo')", "response": "Sets the package s source information if not already set."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the package s concluded licenses.", "response": "def set_pkg_licenses_concluded(self, doc, licenses):\n        \"\"\"Sets the package's concluded licenses.\n        licenses - License info.\n        Raises CardinalityError if already defined.\n        Raises OrderError if no package previously defined.\n        Raises SPDXValueError if data malformed.\n        \"\"\"\n        self.assert_package_exists()\n        if not self.package_conc_lics_set:\n            self.package_conc_lics_set = True\n            if validations.validate_lics_conc(licenses):\n                doc.package.conc_lics = licenses\n                return True\n            else:\n                raise SPDXValueError('Package::ConcludedLicenses')\n        else:\n            raise CardinalityError('Package::ConcludedLicenses')"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_pkg_license_from_file(self, doc, lic):\n        self.assert_package_exists()\n        if validations.validate_lics_from_file(lic):\n            doc.package.licenses_from_files.append(lic)\n            return True\n        else:\n            raise SPDXValueError('Package::LicensesFromFile')", "response": "Adds a license from a file to the package."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsets the package s declared license.", "response": "def set_pkg_license_declared(self, doc, lic):\n        \"\"\"Sets the package's declared license.\n        Raises SPDXValueError if data malformed.\n        Raises OrderError if no package previously defined.\n        Raises CardinalityError if already set.\n        \"\"\"\n        self.assert_package_exists()\n        if not self.package_license_declared_set:\n            self.package_license_declared_set = True\n            if validations.validate_lics_conc(lic):\n                doc.package.license_declared = lic\n                return True\n            else:\n                raise SPDXValueError('Package::LicenseDeclared')\n        else:\n            raise CardinalityError('Package::LicenseDeclared')"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_pkg_license_comment(self, doc, text):\n        self.assert_package_exists()\n        if not self.package_license_comment_set:\n            self.package_license_comment_set = True\n            if validations.validate_pkg_lics_comment(text):\n                doc.package.license_comment = str_from_text(text)\n                return True\n            else:\n                raise SPDXValueError('Package::LicenseComment')\n        else:\n            raise CardinalityError('Package::LicenseComment')", "response": "Sets the package s license comment."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_pkg_cr_text(self, doc, text):\n        self.assert_package_exists()\n        if not self.package_cr_text_set:\n            self.package_cr_text_set = True\n            if validations.validate_pkg_cr_text(text):\n                if isinstance(text, string_types):\n                    doc.package.cr_text = str_from_text(text)\n                else:\n                    doc.package.cr_text = text  # None or NoAssert\n            else:\n                raise SPDXValueError('Package::CopyrightText')\n        else:\n            raise CardinalityError('Package::CopyrightText')", "response": "Sets the copyright text of the package."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset the package summary.", "response": "def set_pkg_summary(self, doc, text):\n        \"\"\"Set's the package summary.\n        Raises SPDXValueError if text is not free form text.\n        Raises CardinalityError if summary already set.\n        Raises OrderError if no package previously defined.\n        \"\"\"\n        self.assert_package_exists()\n        if not self.package_summary_set:\n            self.package_summary_set = True\n            if validations.validate_pkg_summary(text):\n                doc.package.summary = str_from_text(text)\n            else:\n                raise SPDXValueError('Package::Summary')\n        else:\n            raise CardinalityError('Package::Summary')"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_pkg_desc(self, doc, text):\n        self.assert_package_exists()\n        if not self.package_desc_set:\n            self.package_desc_set = True\n            if validations.validate_pkg_desc(text):\n                doc.package.description = str_from_text(text)\n            else:\n                raise SPDXValueError('Package::Description')\n        else:\n            raise CardinalityError('Package::Description')", "response": "Set the package s description."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nset the name of the file instance in the specified doc.", "response": "def set_file_name(self, doc, name):\n        \"\"\"Raises OrderError if no package defined.\n        \"\"\"\n        if self.has_package(doc):\n            doc.package.files.append(file.File(name))\n            # A file name marks the start of a new file instance.\n            # The builder must be reset\n            # FIXME: this state does not make sense\n            self.reset_file_stat()\n            return True\n        else:\n            raise OrderError('File::Name')"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsets the file SPDX Identifier.", "response": "def set_file_spdx_id(self, doc, spdx_id):\n        \"\"\"\n        Sets the file SPDX Identifier.\n        Raises OrderError if no package or no file defined.\n        Raises SPDXValueError if malformed value.\n        Raises CardinalityError if more than one spdx_id set.\n        \"\"\"\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_spdx_id_set:\n                self.file_spdx_id_set = True\n                if validations.validate_file_spdx_id(spdx_id):\n                    self.file(doc).spdx_id = spdx_id\n                    return True\n                else:\n                    raise SPDXValueError('File::SPDXID')\n            else:\n                raise CardinalityError('File::SPDXID')\n        else:\n            raise OrderError('File::SPDXID')"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_file_comment(self, doc, text):\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_comment_set:\n                self.file_comment_set = True\n                if validations.validate_file_comment(text):\n                    self.file(doc).comment = str_from_text(text)\n                    return True\n                else:\n                    raise SPDXValueError('File::Comment')\n            else:\n                raise CardinalityError('File::Comment')\n        else:\n            raise OrderError('File::Comment')", "response": "Sets the comment for a file in the archive."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsetting the type of the file in the archive.", "response": "def set_file_type(self, doc, type_value):\n        \"\"\"\n        Raises OrderError if no package or file defined.\n        Raises CardinalityError if more than one type set.\n        Raises SPDXValueError if type is unknown.\n        \"\"\"\n        type_dict = {\n            'SOURCE': file.FileType.SOURCE,\n            'BINARY': file.FileType.BINARY,\n            'ARCHIVE': file.FileType.ARCHIVE,\n            'OTHER': file.FileType.OTHER\n        }\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_type_set:\n                self.file_type_set = True\n                if type_value in type_dict.keys():\n                    self.file(doc).type = type_dict[type_value]\n                    return True\n                else:\n                    raise SPDXValueError('File::Type')\n            else:\n                raise CardinalityError('File::Type')\n        else:\n            raise OrderError('File::Type')"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_file_chksum(self, doc, chksum):\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_chksum_set:\n                self.file_chksum_set = True\n                self.file(doc).chk_sum = checksum_from_sha1(chksum)\n                return True\n            else:\n                raise CardinalityError('File::CheckSum')\n        else:\n            raise OrderError('File::CheckSum')", "response": "Sets the chksum of a file in the database."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_concluded_license(self, doc, lic):\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_conc_lics_set:\n                self.file_conc_lics_set = True\n                if validations.validate_lics_conc(lic):\n                    self.file(doc).conc_lics = lic\n                    return True\n                else:\n                    raise SPDXValueError('File::ConcludedLicense')\n            else:\n                raise CardinalityError('File::ConcludedLicense')\n        else:\n            raise OrderError('File::ConcludedLicense')", "response": "Sets the concluded license for the file."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef set_file_license_in_file(self, doc, lic):\n        if self.has_package(doc) and self.has_file(doc):\n            if validations.validate_file_lics_in_file(lic):\n                self.file(doc).add_lics(lic)\n                return True\n            else:\n                raise SPDXValueError('File::LicenseInFile')\n        else:\n            raise OrderError('File::LicenseInFile')", "response": "Sets the license in file for the specified document."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nset the license comment for the file.", "response": "def set_file_license_comment(self, doc, text):\n        \"\"\"\n        Raises OrderError if no package or file defined.\n        Raises SPDXValueError if text is not free form text.\n        Raises CardinalityError if more than one per file.\n        \"\"\"\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_license_comment_set:\n                self.file_license_comment_set = True\n                if validations.validate_file_lics_comment(text):\n                    self.file(doc).license_comment = str_from_text(text)\n                else:\n                    raise SPDXValueError('File::LicenseComment')\n            else:\n                raise CardinalityError('File::LicenseComment')\n        else:\n            raise OrderError('File::LicenseComment')"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset copyright text for a file in the order.", "response": "def set_file_copyright(self, doc, text):\n        \"\"\"Raises OrderError if no package or file defined.\n        Raises SPDXValueError if not free form text or NONE or NO_ASSERT.\n        Raises CardinalityError if more than one.\n        \"\"\"\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_copytext_set:\n                self.file_copytext_set = True\n                if validations.validate_file_cpyright(text):\n                    if isinstance(text, string_types):\n                        self.file(doc).copyright = str_from_text(text)\n                    else:\n                        self.file(doc).copyright = text  # None or NoAssert\n                    return True\n                else:\n                    raise SPDXValueError('File::CopyRight')\n            else:\n                raise CardinalityError('File::CopyRight')\n        else:\n            raise OrderError('File::CopyRight')"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets the notice for the file in the article.", "response": "def set_file_notice(self, doc, text):\n        \"\"\"Raises OrderError if no package or file defined.\n        Raises SPDXValueError if not free form text.\n        Raises CardinalityError if more than one.\n        \"\"\"\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_notice_set:\n                self.file_notice_set = True\n                if validations.validate_file_notice(text):\n                    self.file(doc).notice = str_from_text(text)\n                else:\n                    raise SPDXValueError('File::Notice')\n            else:\n                raise CardinalityError('File::Notice')\n        else:\n            raise OrderError('File::Notice')"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef add_file_contribution(self, doc, value):\n        if self.has_package(doc) and self.has_file(doc):\n            self.file(doc).add_contrib(value)\n        else:\n            raise OrderError('File::Contributor')", "response": "Adds a contributor to the order."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef add_file_dep(self, doc, value):\n        if self.has_package(doc) and self.has_file(doc):\n            self.file(doc).add_depend(value)\n        else:\n            raise OrderError('File::Dependency')", "response": "Adds a dependency to the file."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets a file name uri or home artificat. Raises OrderError if no package or file defined.", "response": "def set_file_atrificat_of_project(self, doc, symbol, value):\n        \"\"\"Sets a file name, uri or home artificat.\n        Raises OrderError if no package or file defined.\n        \"\"\"\n        if self.has_package(doc) and self.has_file(doc):\n            self.file(doc).add_artifact(symbol, value)\n        else:\n            raise OrderError('File::Artificat')"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nresets the state of the file stat to the default state.", "response": "def reset_file_stat(self):\n        \"\"\"Resets the builder's state to enable building new files.\"\"\"\n        # FIXME: this state does not make sense\n        self.file_spdx_id_set = False\n        self.file_comment_set = False\n        self.file_type_set = False\n        self.file_chksum_set = False\n        self.file_conc_lics_set = False\n        self.file_license_comment_set = False\n        self.file_notice_set = False\n        self.file_copytext_set = False"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadd a new extracted license to the document.", "response": "def set_lic_id(self, doc, lic_id):\n        \"\"\"Adds a new extracted license to the document.\n        Raises SPDXValueError if data format is incorrect.\n        \"\"\"\n        # FIXME: this state does not make sense\n        self.reset_extr_lics()\n        if validations.validate_extracted_lic_id(lic_id):\n            doc.add_extr_lic(document.ExtractedLicense(lic_id))\n            return True\n        else:\n            raise SPDXValueError('ExtractedLicense::id')"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_lic_text(self, doc, text):\n        if self.has_extr_lic(doc):\n            if not self.extr_text_set:\n                self.extr_text_set = True\n                if validations.validate_is_free_form_text(text):\n                    self.extr_lic(doc).text = str_from_text(text)\n                    return True\n                else:\n                    raise SPDXValueError('ExtractedLicense::text')\n            else:\n                raise CardinalityError('ExtractedLicense::text')\n        else:\n            raise OrderError('ExtractedLicense::text')", "response": "Sets the extracted text of a license."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsets the name of the license in the article.", "response": "def set_lic_name(self, doc, name):\n        \"\"\"Sets license name.\n        Raises SPDXValueError if name is not str or utils.NoAssert\n        Raises OrderError if no license id defined.\n        \"\"\"\n        if self.has_extr_lic(doc):\n            if not self.extr_lic_name_set:\n                self.extr_lic_name_set = True\n                if validations.validate_extr_lic_name(name):\n                    self.extr_lic(doc).full_name = name\n                    return True\n                else:\n                    raise SPDXValueError('ExtractedLicense::Name')\n            else:\n                raise CardinalityError('ExtractedLicense::Name')\n        else:\n            raise OrderError('ExtractedLicense::Name')"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_lic_comment(self, doc, comment):\n        if self.has_extr_lic(doc):\n            if not self.extr_lic_comment_set:\n                self.extr_lic_comment_set = True\n                if validations.validate_is_free_form_text(comment):\n                    self.extr_lic(doc).comment = str_from_text(comment)\n                    return True\n                else:\n                    raise SPDXValueError('ExtractedLicense::comment')\n            else:\n                raise CardinalityError('ExtractedLicense::comment')\n        else:\n            raise OrderError('ExtractedLicense::comment')", "response": "Sets the comment of a license in the article."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds a license cross reference.", "response": "def add_lic_xref(self, doc, ref):\n        \"\"\"Adds a license cross reference.\n        Raises OrderError if no License ID defined.\n        \"\"\"\n        if self.has_extr_lic(doc):\n            self.extr_lic(doc).add_xref(ref)\n            return True\n        else:\n            raise OrderError('ExtractedLicense::CrossRef')"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef reset(self):\n        # FIXME: this state does not make sense\n        self.reset_creation_info()\n        self.reset_document()\n        self.reset_package()\n        self.reset_file_stat()\n        self.reset_reviews()\n        self.reset_annotations()\n        self.reset_extr_lics()", "response": "Resets the builder s state for building new documents."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning an ISO - 8601 representation of a datetime object.", "response": "def datetime_iso_format(date):\n    \"\"\"\n    Return an ISO-8601 representation of a datetime object.\n    \"\"\"\n    return \"{0:0>4}-{1:0>2}-{2:0>2}T{3:0>2}:{4:0>2}:{5:0>2}Z\".format(\n        date.year, date.month, date.day, date.hour,\n        date.minute, date.second)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns a datetime object from an ISO 8601 representation.", "response": "def datetime_from_iso_format(string):\n    \"\"\"\n    Return a datetime object from an iso 8601 representation.\n    Return None if string is non conforming.\n    \"\"\"\n    match = DATE_ISO_REGEX.match(string)\n    if match:\n        date = datetime.datetime(year=int(match.group(DATE_ISO_YEAR_GRP)),\n                                 month=int(match.group(DATE_ISO_MONTH_GRP)),\n                                 day=int(match.group(DATE_ISO_DAY_GRP)),\n                                 hour=int(match.group(DATE_ISO_HOUR_GRP)),\n                                 second=int(match.group(DATE_ISO_SEC_GRP)),\n                                 minute=int(match.group(DATE_ISO_MIN_GRP)))\n        return date\n    else:\n        return None"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef build(self, **kwargs):\n        self.yacc = yacc.yacc(module=self, **kwargs)", "response": "Build a YACC object."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse(self, data):\n        try:\n            return self.yacc.parse(data, lexer=self.lex)\n        except:\n            return None", "response": "Parses a license list and returns a License or None."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef write_document(document, out, validate=True):\n    \n    if validate:\n        messages = []\n        messages = document.validate(messages)\n        if messages:\n            raise InvalidDocumentError(messages)\n\n    writer = Writer(document, out)\n    writer.write()", "response": "Write a SPDX RDF document to out."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_checksum_node(self, chksum):\n        chksum_node = BNode()\n        type_triple = (chksum_node, RDF.type, self.spdx_namespace.Checksum)\n        self.graph.add(type_triple)\n        algorithm_triple = (chksum_node, self.spdx_namespace.algorithm, Literal(chksum.identifier))\n        self.graph.add(algorithm_triple)\n        value_triple = (chksum_node, self.spdx_namespace.checksumValue, Literal(chksum.value))\n        self.graph.add(value_triple)\n        return chksum_node", "response": "Create a node representing the checksum of the current skeleton."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nconverts a value to a proper spdx term or Literal.", "response": "def to_special_value(self, value):\n        \"\"\"\n        Return proper spdx term or Literal\n        \"\"\"\n        if isinstance(value, utils.NoAssert):\n            return self.spdx_namespace.noassertion\n        elif isinstance(value, utils.SPDXNone):\n            return self.spdx_namespace.none\n        else:\n            return Literal(value)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ntraversing conjunctions and disjunctions like trees and return a set of all licenses in that tree.", "response": "def licenses_from_tree(self, tree):\n        \"\"\"\n        Traverse conjunctions and disjunctions like trees and return a\n        set of all licenses in it as nodes.\n        \"\"\"\n        # FIXME: this is unordered!\n        licenses = set()\n        self.licenses_from_tree_helper(tree, licenses)\n        return licenses"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a node representing a conjunction of licenses.", "response": "def create_conjunction_node(self, conjunction):\n        \"\"\"\n        Return a node representing a conjunction of licenses.\n        \"\"\"\n        node = BNode()\n        type_triple = (node, RDF.type, self.spdx_namespace.ConjunctiveLicenseSet)\n        self.graph.add(type_triple)\n        licenses = self.licenses_from_tree(conjunction)\n        for lic in licenses:\n            member_triple = (node, self.spdx_namespace.member, lic)\n            self.graph.add(member_triple)\n        return node"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef create_disjunction_node(self, disjunction):\n        node = BNode()\n        type_triple = (node, RDF.type, self.spdx_namespace.DisjunctiveLicenseSet)\n        self.graph.add(type_triple)\n        licenses = self.licenses_from_tree(disjunction)\n        for lic in licenses:\n            member_triple = (node, self.spdx_namespace.member, lic)\n            self.graph.add(member_triple)\n        return node", "response": "Create a disjunction node representing a licenses set."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef create_license_helper(self, lic):\n        if isinstance(lic, document.ExtractedLicense):\n            return self.create_extracted_license(lic)\n        if lic.identifier.rstrip('+') in config.LICENSE_MAP:\n            return URIRef(lic.url)\n        else:\n            matches = [l for l in self.document.extracted_licenses if l.identifier == lic.identifier]\n            if len(matches) != 0:\n                return self.create_extracted_license(matches[0])\n            else:\n                raise InvalidDocumentError('Missing extracted license: {0}'.format(lic.identifier))", "response": "Create a node from a single or multiple licenses."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef create_extracted_license(self, lic):\n        licenses = list(self.graph.triples((None, self.spdx_namespace.licenseId, lic.identifier)))\n        if len(licenses) != 0:\n            return licenses[0][0]  # return subject in first triple\n        else:\n            license_node = BNode()\n            type_triple = (license_node, RDF.type, self.spdx_namespace.ExtractedLicensingInfo)\n            self.graph.add(type_triple)\n            ident_triple = (license_node, self.spdx_namespace.licenseId, Literal(lic.identifier))\n            self.graph.add(ident_triple)\n            text_triple = (license_node, self.spdx_namespace.extractedText, Literal(lic.text))\n            self.graph.add(text_triple)\n            if lic.full_name is not None:\n                name_triple = (license_node, self.spdx_namespace.licenseName, self.to_special_value(lic.full_name))\n                self.graph.add(name_triple)\n            for ref in lic.cross_ref:\n                triple = (license_node, RDFS.seeAlso, URIRef(ref))\n                self.graph.add(triple)\n            if lic.comment is not None:\n                comment_triple = (license_node, RDFS.comment, Literal(lic.comment))\n                self.graph.add(comment_triple)\n            return license_node", "response": "Create the extracted license."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a node representing a license.", "response": "def create_license_node(self, lic):\n        \"\"\"\n        Return a node representing a license.\n        Could be a single license (extracted or part of license list.) or\n        a conjunction/disjunction of licenses.\n        \"\"\"\n        if isinstance(lic, document.LicenseConjunction):\n            return self.create_conjunction_node(lic)\n        elif isinstance(lic, document.LicenseDisjunction):\n            return self.create_disjunction_node(lic)\n        else:\n            return self.create_license_helper(lic)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning the term for the license or the special value if the license is special.", "response": "def license_or_special(self, lic):\n        \"\"\"\n        Check for special values spdx:none and spdx:noassertion.\n        Return the term for the special value or the result of passing\n        license to create_license_node.\n        \"\"\"\n        if isinstance(lic, utils.NoAssert):\n            return self.spdx_namespace.noassertion\n        elif isinstance(lic, utils.SPDXNone):\n            return self.spdx_namespace.none\n        else:\n            return self.create_license_node(lic)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating a node for the file.", "response": "def create_file_node(self, doc_file):\n        \"\"\"\n        Create a node for spdx.file.\n        \"\"\"\n        file_node = URIRef('http://www.spdx.org/files#{id}'.format(\n            id=str(doc_file.spdx_id)))\n        type_triple = (file_node, RDF.type, self.spdx_namespace.File)\n        self.graph.add(type_triple)\n\n        name_triple = (file_node, self.spdx_namespace.fileName, Literal(doc_file.name))\n        self.graph.add(name_triple)\n\n        if doc_file.has_optional_field('comment'):\n            comment_triple = (file_node, RDFS.comment, Literal(doc_file.comment))\n            self.graph.add(comment_triple)\n\n        if doc_file.has_optional_field('type'):\n            ftype = self.spdx_namespace[self.FILE_TYPES[doc_file.type]]\n            ftype_triple = (file_node, self.spdx_namespace.fileType, ftype)\n            self.graph.add(ftype_triple)\n\n        self.graph.add((file_node, self.spdx_namespace.checksum, self.create_checksum_node(doc_file.chk_sum)))\n\n        conc_lic_node = self.license_or_special(doc_file.conc_lics)\n        conc_lic_triple = (file_node, self.spdx_namespace.licenseConcluded, conc_lic_node)\n        self.graph.add(conc_lic_triple)\n\n        license_info_nodes = map(self.license_or_special, doc_file.licenses_in_file)\n        for lic in license_info_nodes:\n            triple = (file_node, self.spdx_namespace.licenseInfoInFile, lic)\n            self.graph.add(triple)\n\n        if doc_file.has_optional_field('license_comment'):\n            comment_triple = (file_node, self.spdx_namespace.licenseComments, Literal(doc_file.license_comment))\n            self.graph.add(comment_triple)\n\n        cr_text_node = self.to_special_value(doc_file.copyright)\n        cr_text_triple = (file_node, self.spdx_namespace.copyrightText, cr_text_node)\n        self.graph.add(cr_text_triple)\n\n        if doc_file.has_optional_field('notice'):\n            notice_triple = (file_node, self.spdx_namespace.noticeText, doc_file.notice)\n            self.graph.add(notice_triple)\n\n        contrib_nodes = map(lambda c: Literal(c), doc_file.contributors)\n        contrib_triples = [(file_node, self.spdx_namespace.fileContributor, node) for node in contrib_nodes]\n        for triple in contrib_triples:\n            self.graph.add(triple)\n\n        return file_node"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef add_file_dependencies_helper(self, doc_file):\n        subj_triples = list(self.graph.triples((None, self.spdx_namespace.fileName, Literal(doc_file.name))))\n        if len(subj_triples) != 1:\n            raise InvalidDocumentError('Could not find dependency subject {0}'.format(doc_file.name))\n        subject_node = subj_triples[0][0]\n        for dependency in doc_file.dependencies:\n            dep_triples = list(self.graph.triples((None, self.spdx_namespace.fileName, Literal(dependency))))\n            if len(dep_triples) == 1:\n                dep_node = dep_triples[0][0]\n                dep_triple = (subject_node, self.spdx_namespace.fileDependency, dep_node)\n                self.graph.add(dep_triple)\n            else:\n                print('Warning could not resolve file dependency {0} -> {1}'.format(doc_file.name, dependency))", "response": "Add all the file dependencies for a single file."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncreates a review node.", "response": "def create_review_node(self, review):\n        \"\"\"\n        Return a review node.\n        \"\"\"\n        review_node = BNode()\n        type_triple = (review_node, RDF.type, self.spdx_namespace.Review)\n        self.graph.add(type_triple)\n\n        reviewer_node = Literal(review.reviewer.to_value())\n        self.graph.add((review_node, self.spdx_namespace.reviewer, reviewer_node))\n        reviewed_date_node = Literal(review.review_date_iso_format)\n        reviewed_triple = (review_node, self.spdx_namespace.reviewDate, reviewed_date_node)\n        self.graph.add(reviewed_triple)\n        if review.has_comment:\n            comment_node = Literal(review.comment)\n            comment_triple = (review_node, RDFS.comment, comment_node)\n            self.graph.add(comment_triple)\n\n        return review_node"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ncreating an annotation node.", "response": "def create_annotation_node(self, annotation):\n        \"\"\"\n        Return an annotation node.\n        \"\"\"\n        annotation_node = URIRef(str(annotation.spdx_id))\n        type_triple = (annotation_node, RDF.type, self.spdx_namespace.Annotation)\n        self.graph.add(type_triple)\n\n        annotator_node = Literal(annotation.annotator.to_value())\n        self.graph.add((annotation_node, self.spdx_namespace.annotator, annotator_node))\n        annotation_date_node = Literal(annotation.annotation_date_iso_format)\n        annotation_triple = (annotation_node, self.spdx_namespace.annotationDate, annotation_date_node)\n        self.graph.add(annotation_triple)\n        if annotation.has_comment:\n            comment_node = Literal(annotation.comment)\n            comment_triple = (annotation_node, RDFS.comment, comment_node)\n            self.graph.add(comment_triple)\n        annotation_type_node = Literal(annotation.annotation_type)\n        annotation_type_triple = (annotation_node, self.spdx_namespace.annotationType, annotation_type_node)\n        self.graph.add(annotation_type_triple)\n\n        return annotation_node"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef creators(self):\n        return map(lambda c: Literal(c.to_value()), self.document.creation_info.creators)", "response": "Return a list of creator nodes."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nadding and return a creation info node to the graph", "response": "def create_creation_info(self):\n        \"\"\"\n        Add and return a creation info node to graph\n        \"\"\"\n        ci_node = BNode()\n        # Type property\n        type_triple = (ci_node, RDF.type, self.spdx_namespace.CreationInfo)\n        self.graph.add(type_triple)\n\n        created_date = Literal(self.document.creation_info.created_iso_format)\n        created_triple = (ci_node, self.spdx_namespace.created, created_date)\n        self.graph.add(created_triple)\n\n        creators = self.creators()\n        for creator in creators:\n            self.graph.add((ci_node, self.spdx_namespace.creator, creator))\n\n        if self.document.creation_info.has_comment:\n            comment_node = Literal(self.document.creation_info.comment)\n            comment_triple = (ci_node, RDFS.comment, comment_node)\n            self.graph.add(comment_triple)\n\n        return ci_node"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef create_external_document_ref_node(self, ext_document_references):\n        ext_doc_ref_node = BNode()\n        type_triple = (ext_doc_ref_node, RDF.type, self.spdx_namespace.ExternalDocumentRef)\n        self.graph.add(type_triple)\n\n        ext_doc_id = Literal(\n            ext_document_references.external_document_id)\n        ext_doc_id_triple = (\n            ext_doc_ref_node, self.spdx_namespace.externalDocumentId, ext_doc_id)\n        self.graph.add(ext_doc_id_triple)\n\n        doc_uri = Literal(\n            ext_document_references.spdx_document_uri)\n        doc_uri_triple = (\n            ext_doc_ref_node, self.spdx_namespace.spdxDocument, doc_uri)\n        self.graph.add(doc_uri_triple)\n\n        checksum_node = self.create_checksum_node(\n            ext_document_references.check_sum)\n        self.graph.add(\n            (ext_doc_ref_node, self.spdx_namespace.checksum, checksum_node))\n\n        return ext_doc_ref_node", "response": "Add and return a creation info node to the graph"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a node representing the verification code of a package.", "response": "def package_verif_node(self, package):\n        \"\"\"\n        Return a node representing package verification code.\n        \"\"\"\n        verif_node = BNode()\n        type_triple = (verif_node, RDF.type, self.spdx_namespace.PackageVerificationCode)\n        self.graph.add(type_triple)\n        value_triple = (verif_node, self.spdx_namespace.packageVerificationCodeValue, Literal(package.verif_code))\n        self.graph.add(value_triple)\n        excl_file_nodes = map(\n            lambda excl: Literal(excl), package.verif_exc_files)\n        excl_predicate = self.spdx_namespace.packageVerificationCodeExcludedFile\n        excl_file_triples = [(verif_node, excl_predicate, xcl_file) for xcl_file in excl_file_nodes]\n        for trp in excl_file_triples:\n            self.graph.add(trp)\n        return verif_node"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef handle_package_literal_optional(self, package, package_node, predicate, field):\n        if package.has_optional_field(field):\n            value = getattr(package, field, None)\n            value_node = self.to_special_value(value)\n            triple = (package_node, predicate, value_node)\n            self.graph.add(triple)", "response": "Handle optional field in package."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nhandles optional fields in the package node.", "response": "def handle_pkg_optional_fields(self, package, package_node):\n        \"\"\"\n        Write package optional fields.\n        \"\"\"\n        self.handle_package_literal_optional(package, package_node, self.spdx_namespace.versionInfo, 'version')\n        self.handle_package_literal_optional(package, package_node, self.spdx_namespace.packageFileName, 'file_name')\n        self.handle_package_literal_optional(package, package_node, self.spdx_namespace.supplier, 'supplier')\n        self.handle_package_literal_optional(package, package_node, self.spdx_namespace.originator, 'originator')\n        self.handle_package_literal_optional(package, package_node, self.spdx_namespace.sourceInfo, 'source_info')\n        self.handle_package_literal_optional(package, package_node, self.spdx_namespace.licenseComments, 'license_comment')\n        self.handle_package_literal_optional(package, package_node, self.spdx_namespace.summary, 'summary')\n        self.handle_package_literal_optional(package, package_node, self.spdx_namespace.description, 'description')\n\n        if package.has_optional_field('check_sum'):\n            checksum_node = self.create_checksum_node(package.check_sum)\n            self.graph.add((package_node, self.spdx_namespace.checksum, checksum_node))\n\n        if package.has_optional_field('homepage'):\n            homepage_node = URIRef(self.to_special_value(package.homepage))\n            homepage_triple = (package_node, self.doap_namespace.homepage, homepage_node)\n            self.graph.add(homepage_triple)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncreate a node representing the package.", "response": "def create_package_node(self, package):\n        \"\"\"\n        Return a Node representing the package.\n        Files must have been added to the graph before this method is called.\n        \"\"\"\n        package_node = BNode()\n        type_triple = (package_node, RDF.type, self.spdx_namespace.Package)\n        self.graph.add(type_triple)\n        # Handle optional fields:\n        self.handle_pkg_optional_fields(package, package_node)\n        # package name\n        name_triple = (package_node, self.spdx_namespace.name, Literal(package.name))\n        self.graph.add(name_triple)\n        # Package download location\n        down_loc_node = (package_node, self.spdx_namespace.downloadLocation, self.to_special_value(package.download_location))\n        self.graph.add(down_loc_node)\n        # Handle package verification\n        verif_node = self.package_verif_node(package)\n        verif_triple = (package_node, self.spdx_namespace.packageVerificationCode, verif_node)\n        self.graph.add(verif_triple)\n        # Handle concluded license\n        conc_lic_node = self.license_or_special(package.conc_lics)\n        conc_lic_triple = (package_node, self.spdx_namespace.licenseConcluded, conc_lic_node)\n        self.graph.add(conc_lic_triple)\n        # Handle declared license\n        decl_lic_node = self.license_or_special(package.license_declared)\n        decl_lic_triple = (package_node, self.spdx_namespace.licenseDeclared, decl_lic_node)\n        self.graph.add(decl_lic_triple)\n        # Package licenses from files\n        licenses_from_files_nodes = map(lambda el: self.license_or_special(el), package.licenses_from_files)\n        lic_from_files_predicate = self.spdx_namespace.licenseInfoFromFiles\n        lic_from_files_triples = [(package_node, lic_from_files_predicate, node) for node in licenses_from_files_nodes]\n        for triple in lic_from_files_triples:\n            self.graph.add(triple)\n        # Copyright Text\n        cr_text_node = self.to_special_value(package.cr_text)\n        cr_text_triple = (package_node, self.spdx_namespace.copyrightText, cr_text_node)\n        self.graph.add(cr_text_triple)\n        # Handle files\n        self.handle_package_has_file(package, package_node)\n        return package_node"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nhandles the case where a package has a file.", "response": "def handle_package_has_file_helper(self, pkg_file):\n        \"\"\"\n        Return node representing pkg_file\n        pkg_file should be instance of spdx.file.\n        \"\"\"\n        nodes = list(self.graph.triples((None, self.spdx_namespace.fileName, Literal(pkg_file.name))))\n        if len(nodes) == 1:\n            return nodes[0][0]\n        else:\n            raise InvalidDocumentError('handle_package_has_file_helper could not' +\n                                       ' find file node for file: {0}'.format(pkg_file.name))"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nhandles hasFile nodes in package.", "response": "def handle_package_has_file(self, package, package_node):\n        \"\"\"\n        Add hasFile triples to graph.\n        Must be called after files have been added.\n        \"\"\"\n        file_nodes = map(self.handle_package_has_file_helper, package.files)\n        triples = [(package_node, self.spdx_namespace.hasFile, node) for node in file_nodes]\n        for triple in triples:\n            self.graph.add(triple)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates the root document node and return it.", "response": "def create_doc(self):\n        \"\"\"\n        Add and return the root document node to graph.\n        \"\"\"\n        doc_node = URIRef('http://www.spdx.org/tools#SPDXRef-DOCUMENT')\n        # Doc type\n        self.graph.add((doc_node, RDF.type, self.spdx_namespace.SpdxDocument))\n        # Version\n        vers_literal = Literal(str(self.document.version))\n        self.graph.add((doc_node, self.spdx_namespace.specVersion, vers_literal))\n        # Data license\n        data_lics = URIRef(self.document.data_license.url)\n        self.graph.add((doc_node, self.spdx_namespace.dataLicense, data_lics))\n        doc_name = URIRef(self.document.name)\n        self.graph.add((doc_node, self.spdx_namespace.name, doc_name))\n        return doc_node"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef load_license_list(file_name):\n    licenses_map = {}\n    with codecs.open(file_name, 'rb', encoding='utf-8') as lics:\n        licenses = json.load(lics)\n        version = licenses['licenseListVersion'].split('.')\n        for lic in licenses['licenses']:\n            if lic.get('isDeprecatedLicenseId'):\n                continue\n            name = lic['name']\n            identifier = lic['licenseId']\n            licenses_map[name] = identifier\n            licenses_map[identifier] = name\n    return version, licenses_map", "response": "Load the licenses list from a JSON file and return the version tuple and a mapping of name - > id - > name"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef validate(self, messages):\n        messages = self.validate_creators(messages)\n        messages = self.validate_created(messages)\n\n        return messages", "response": "Returns True if the fields are valid according to the SPDX standard."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef more_than_one_error(self, field):\n        msg = 'More than one {0} defined.'.format(field)\n        self.logger.log(msg)\n        self.error = True", "response": "Logs a more than one error."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef value_error(self, key, bad_value):\n        msg = ERROR_MESSAGES[key].format(bad_value)\n        self.logger.log(msg)\n        self.error = True", "response": "Reports a value error using ERROR_MESSAGES dict."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nchecking if value is a special SPDX value such as NONE NOASSERTION or UNKNOWN returns proper model. Otherwise returns value", "response": "def to_special_value(self, value):\n        \"\"\"Checks if value is a special SPDX value such as\n        NONE, NOASSERTION or UNKNOWN if so returns proper model.\n        else returns value\"\"\"\n        if value == self.spdx_namespace.none:\n            return utils.SPDXNone()\n        elif value == self.spdx_namespace.noassertion:\n            return utils.NoAssert()\n        elif value == self.spdx_namespace.unknown:\n            return utils.UnKnown()\n        else:\n            return value"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nhandle a license from a lics resource.", "response": "def handle_lics(self, lics):\n        \"\"\"\n        Return a License from a `lics` license resource.\n        \"\"\"\n        # Handle extracted licensing info type.\n        if (lics, RDF.type, self.spdx_namespace['ExtractedLicensingInfo']) in self.graph:\n            return self.parse_only_extr_license(lics)\n\n        # Assume resource, hence the path separator\n        ident_start = lics.rfind('/') + 1\n        if ident_start == 0:\n            # special values such as spdx:noassertion\n            special = self.to_special_value(lics)\n            if special == lics:\n                if self.LICS_REF_REGEX.match(lics):\n                    # Is a license ref i.e LicenseRef-1\n                    return document.License.from_identifier(lics)\n                else:\n                    # Not a known license form\n                    raise SPDXValueError('License')\n            else:\n                # is a special value\n                return special\n        else:\n            # license url\n            return document.License.from_identifier(lics[ident_start:])"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef get_extr_license_ident(self, extr_lic):\n        identifier_tripples = list(self.graph.triples((extr_lic, self.spdx_namespace['licenseId'], None)))\n\n        if not identifier_tripples:\n            self.error = True\n            msg = 'Extracted license must have licenseId property.'\n            self.logger.log(msg)\n            return\n\n        if len(identifier_tripples) > 1:\n            self.more_than_one_error('extracted license identifier_tripples')\n            return\n\n        identifier_tripple = identifier_tripples[0]\n        _s, _p, identifier = identifier_tripple\n        return identifier", "response": "Get an a license identifier from an ExtractedLicense or None."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets extracted text from an ExtractedLicense object.", "response": "def get_extr_license_text(self, extr_lic):\n        \"\"\"\n        Return extracted text  from an ExtractedLicense or None.\n        \"\"\"\n        text_tripples = list(self.graph.triples((extr_lic, self.spdx_namespace['extractedText'], None)))\n        if not text_tripples:\n            self.error = True\n            msg = 'Extracted license must have extractedText property'\n            self.logger.log(msg)\n            return\n\n        if len(text_tripples) > 1:\n            self.more_than_one_error('extracted license text')\n            return\n\n        text_tripple = text_tripples[0]\n        _s, _p, text = text_tripple\n        return text"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_extr_lic_name(self, extr_lic):\n        extr_name_list = list(self.graph.triples((extr_lic, self.spdx_namespace['licenseName'], None)))\n        if len(extr_name_list) > 1:\n            self.more_than_one_error('extracted license name')\n            return\n        elif len(extr_name_list) == 0:\n            return\n        return self.to_special_value(extr_name_list[0][2])", "response": "Get the ExtractedLicense name from an ExtractedLicense object or None."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list of cross references for the given LIC.", "response": "def get_extr_lics_xref(self, extr_lic):\n        \"\"\"\n        Return a list of cross references.\n        \"\"\"\n        xrefs = list(self.graph.triples((extr_lic, RDFS.seeAlso, None)))\n        return map(lambda xref_triple: xref_triple[2], xrefs)"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ngets the extracted license comment from the extraneous lics.", "response": "def get_extr_lics_comment(self, extr_lics):\n        \"\"\"\n        Return license comment or None.\n        \"\"\"\n        comment_list = list(self.graph.triples(\n            (extr_lics, RDFS.comment, None)))\n        if len(comment_list) > 1 :\n            self.more_than_one_error('extracted license comment')\n            return\n        elif len(comment_list) == 1:\n            return comment_list[0][2]\n        else:\n            return"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef parse_only_extr_license(self, extr_lic):\n        # Grab all possible values\n        ident = self.get_extr_license_ident(extr_lic)\n        text = self.get_extr_license_text(extr_lic)\n        comment = self.get_extr_lics_comment(extr_lic)\n        xrefs = self.get_extr_lics_xref(extr_lic)\n        name = self.get_extr_lic_name(extr_lic)\n\n        if not ident:\n            # Must have identifier\n            return\n\n        # Set fields\n        # FIXME: the constructor of the license should alwas accept a name\n        lic = document.ExtractedLicense(ident)\n        if text is not None:\n            lic.text = text\n        if name is not None:\n            lic.full_name = name\n        if comment is not None:\n            lic.comment = comment\n        lic.cross_ref = map(lambda x: six.text_type(x), xrefs)\n        return lic", "response": "Parse a ExtractedLicense object that is only relevant to a single license."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nbuild and return an ExtractedLicense or None.", "response": "def handle_extracted_license(self, extr_lic):\n        \"\"\"\n        Build and return an ExtractedLicense or None.\n        Note that this function adds the license to the document.\n        \"\"\"\n        lic = self.parse_only_extr_license(extr_lic)\n        if lic is not None:\n            self.doc.add_extr_lic(lic)\n        return lic"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nhandle a list of licenses and returns a license representing a cls object.", "response": "def _handle_license_list(self, lics_set, cls=None):\n        \"\"\"\n        Return a license representing a `cls` object (LicenseConjunction\n        or LicenseDisjunction) from a list of license resources or None.\n        \"\"\"\n        licenses = []\n        for _, _, lics_member in self.graph.triples(\n            (lics_set, self.spdx_namespace['member'], None)):\n            try:\n                if (lics_member, RDF.type, self.spdx_namespace['ExtractedLicensingInfo']) in self.graph:\n                    lics = self.handle_extracted_license(lics_member)\n                    if lics is not None:\n                        licenses.append(lics)\n                else:\n                    licenses.append(self.handle_lics(lics_member))\n            except CardinalityError:\n                self.value_error('LICS_LIST_MEMBER', lics_member)\n                break\n        if len(licenses) > 1:\n            return reduce(lambda a, b: cls(a, b), licenses)\n        else:\n            self.value_error('PKG_CONC_LIST', '')\n            return"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nparsing the package fields.", "response": "def parse_package(self, p_term):\n        \"\"\"Parses package fields.\"\"\"\n        # Check there is a pacakge name\n        if not (p_term, self.spdx_namespace['name'], None) in self.graph:\n            self.error = True\n            self.logger.log('Package must have a name.')\n            # Create dummy package so that we may continue parsing the rest of\n            # the package fields.\n            self.builder.create_package(self.doc, 'dummy_package')\n        else:\n            for _s, _p, o in self.graph.triples((p_term, self.spdx_namespace['name'], None)):\n                try:\n                    self.builder.create_package(self.doc, six.text_type(o))\n                except CardinalityError:\n                    self.more_than_one_error('Package name')\n                    break\n\n        self.p_pkg_vinfo(p_term, self.spdx_namespace['versionInfo'])\n        self.p_pkg_fname(p_term, self.spdx_namespace['packageFileName'])\n        self.p_pkg_suppl(p_term, self.spdx_namespace['supplier'])\n        self.p_pkg_originator(p_term, self.spdx_namespace['originator'])\n        self.p_pkg_down_loc(p_term, self.spdx_namespace['downloadLocation'])\n        self.p_pkg_homepg(p_term, self.doap_namespace['homepage'])\n        self.p_pkg_chk_sum(p_term, self.spdx_namespace['checksum'])\n        self.p_pkg_src_info(p_term, self.spdx_namespace['sourceInfo'])\n        self.p_pkg_verif_code(p_term, self.spdx_namespace['packageVerificationCode'])\n        self.p_pkg_lic_conc(p_term, self.spdx_namespace['licenseConcluded'])\n        self.p_pkg_lic_decl(p_term, self.spdx_namespace['licenseDeclared'])\n        self.p_pkg_lics_info_from_files(p_term, self.spdx_namespace['licenseInfoFromFiles'])\n        self.p_pkg_comments_on_lics(p_term, self.spdx_namespace['licenseComments'])\n        self.p_pkg_cr_text(p_term, self.spdx_namespace['copyrightText'])\n        self.p_pkg_summary(p_term, self.spdx_namespace['summary'])\n        self.p_pkg_descr(p_term, self.spdx_namespace['description'])"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef handle_pkg_lic(self, p_term, predicate, builder_func):\n        try:\n            for _, _, licenses in self.graph.triples((p_term, predicate, None)):\n                if (licenses, RDF.type, self.spdx_namespace['ConjunctiveLicenseSet']) in self.graph:\n                    lics = self.handle_conjunctive_list(licenses)\n                    builder_func(self.doc, lics)\n\n                elif (licenses, RDF.type, self.spdx_namespace['DisjunctiveLicenseSet']) in self.graph:\n                    lics = self.handle_disjunctive_list(licenses)\n                    builder_func(self.doc, lics)\n\n                else:\n                    try:\n                        lics = self.handle_lics(licenses)\n                        builder_func(self.doc, lics)\n                    except SPDXValueError:\n                        self.value_error('PKG_SINGLE_LICS', licenses)\n        except CardinalityError:\n            self.more_than_one_error('package {0}'.format(predicate))", "response": "Handles package lics concluded or declared."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn first found fileName property or None.", "response": "def get_file_name(self, f_term):\n        \"\"\"Returns first found fileName property or None if not found.\"\"\"\n        for _, _, name in self.graph.triples((f_term, self.spdx_namespace['fileName'], None)):\n            return name\n        return"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nparsing all file contributors and add them to the model.", "response": "def p_file_contributor(self, f_term, predicate):\n        \"\"\"\n        Parse all file contributors and adds them to the model.\n        \"\"\"\n        for _, _, contributor in self.graph.triples((f_term, predicate, None)):\n            self.builder.add_file_contribution(self.doc, six.text_type(contributor))"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset file notice text.", "response": "def p_file_notice(self, f_term, predicate):\n        \"\"\"Sets file notice text.\"\"\"\n        try:\n            for _, _, notice in self.graph.triples((f_term, predicate, None)):\n                self.builder.set_file_notice(self.doc, six.text_type(notice))\n        except CardinalityError:\n            self.more_than_one_error('file notice')"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting file comment text.", "response": "def p_file_comment(self, f_term, predicate):\n        \"\"\"Sets file comment text.\"\"\"\n        try:\n            for _, _, comment in self.graph.triples((f_term, predicate, None)):\n                self.builder.set_file_comment(self.doc, six.text_type(comment))\n        except CardinalityError:\n            self.more_than_one_error('file comment')"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nhandling file artifactOf. Note: does not handle artifact of project URI.", "response": "def p_file_artifact(self, f_term, predicate):\n        \"\"\"Handles file artifactOf.\n        Note: does not handle artifact of project URI.\n        \"\"\"\n        for _, _, project in self.graph.triples((f_term, predicate, None)):\n            if (project, RDF.type, self.doap_namespace['Project']):\n                self.p_file_project(project)\n            else:\n                self.error = True\n                msg = 'File must be artifact of doap:Project'\n                self.logger.log(msg)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef p_file_project(self, project):\n        for _, _, name in self.graph.triples((project, self.doap_namespace['name'], None)):\n            self.builder.set_file_atrificat_of_project(self.doc, 'name', six.text_type(name))\n        for _, _, homepage in self.graph.triples(\n            (project, self.doap_namespace['homepage'], None)):\n            self.builder.set_file_atrificat_of_project(self.doc, 'home', six.text_type(homepage))", "response": "Parse the doap project name and homepage and set them using the file builder."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef p_file_cr_text(self, f_term, predicate):\n        try:\n            for _, _, cr_text in self.graph.triples((f_term, predicate, None)):\n                self.builder.set_file_copyright(self.doc, six.text_type(cr_text))\n        except CardinalityError:\n            self.more_than_one_error('file copyright text')", "response": "Sets file copyright text."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting file license comments on licensing.", "response": "def p_file_comments_on_lics(self, f_term, predicate):\n        \"\"\"Sets file license comment.\"\"\"\n        try:\n            for _, _, comment in self.graph.triples((f_term, predicate, None)):\n                self.builder.set_file_license_comment(self.doc, six.text_type(comment))\n        except CardinalityError:\n            self.more_than_one_error('file comments on license')"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nset file license information.", "response": "def p_file_lic_info(self, f_term, predicate):\n        \"\"\"Sets file license information.\"\"\"\n        for _, _, info in self.graph.triples((f_term, predicate, None)):\n            lic = self.handle_lics(info)\n            if lic is not None:\n                self.builder.set_file_license_in_file(self.doc, lic)"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets the file type of the file.", "response": "def p_file_type(self, f_term, predicate):\n        \"\"\"Sets file type.\"\"\"\n        try:\n            for _, _, ftype in self.graph.triples((f_term, predicate, None)):\n                try:\n                    if ftype.endswith('binary'):\n                        ftype = 'BINARY'\n                    elif ftype.endswith('source'):\n                        ftype = 'SOURCE'\n                    elif ftype.endswith('other'):\n                        ftype = 'OTHER'\n                    elif ftype.endswith('archive'):\n                        ftype = 'ARCHIVE'\n                    self.builder.set_file_type(self.doc, ftype)\n                except SPDXValueError:\n                    self.value_error('FILE_TYPE', ftype)\n        except CardinalityError:\n            self.more_than_one_error('file type')"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting file checksum. Assumes SHA1 algorithm without checking.", "response": "def p_file_chk_sum(self, f_term, predicate):\n        \"\"\"Sets file checksum. Assumes SHA1 algorithm without checking.\"\"\"\n        try:\n            for _s, _p, checksum in self.graph.triples((f_term, predicate, None)):\n                for _, _, value in self.graph.triples((checksum, self.spdx_namespace['checksumValue'], None)):\n                    self.builder.set_file_chksum(self.doc, six.text_type(value))\n        except CardinalityError:\n            self.more_than_one_error('File checksum')"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef p_file_lic_conc(self, f_term, predicate):\n        try:\n            for _, _, licenses in self.graph.triples((f_term, predicate, None)):\n                if (licenses, RDF.type, self.spdx_namespace['ConjunctiveLicenseSet']) in self.graph:\n                    lics = self.handle_conjunctive_list(licenses)\n                    self.builder.set_concluded_license(self.doc, lics)\n\n                elif (licenses, RDF.type, self.spdx_namespace['DisjunctiveLicenseSet']) in self.graph:\n                    lics = self.handle_disjunctive_list(licenses)\n                    self.builder.set_concluded_license(self.doc, lics)\n\n                else:\n                    try:\n                        lics = self.handle_lics(licenses)\n                        self.builder.set_concluded_license(self.doc, lics)\n                    except SPDXValueError:\n                        self.value_error('FILE_SINGLE_LICS', licenses)\n        except CardinalityError:\n            self.more_than_one_error('file {0}'.format(predicate))", "response": "Sets file licenses concluded."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_review_date(self, r_term):\n        reviewed_list = list(self.graph.triples((r_term, self.spdx_namespace['reviewDate'], None)))\n        if len(reviewed_list) != 1:\n            self.error = True\n            msg = 'Review must have exactlyone review date'\n            self.logger.log(msg)\n            return\n        return six.text_type(reviewed_list[0][2])", "response": "Returns the review date of the object."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nreturn the reviewer as creator object or None if failed.", "response": "def get_reviewer(self, r_term):\n        \"\"\"Returns reviewer as creator object or None if failed.\n        Reports errors on failure.\n        \"\"\"\n        reviewer_list = list(self.graph.triples((r_term, self.spdx_namespace['reviewer'], None)))\n        if len(reviewer_list) != 1:\n            self.error = True\n            msg = 'Review must have exactly one reviewer'\n            self.logger.log(msg)\n            return\n        try:\n            return self.builder.create_entity(self.doc, six.text_type(reviewer_list[0][2]))\n        except SPDXValueError:\n            self.value_error('REVIEWER_VALUE', reviewer_list[0][2])"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef get_annotation_type(self, r_term):\n        for _, _, typ in self.graph.triples((\n                r_term, self.spdx_namespace['annotationType'], None)):\n            if typ is not None:\n                return typ\n            else:\n                self.error = True\n                msg = 'Annotation must have exactly one annotation type.'\n                self.logger.log(msg)\n                return", "response": "Returns the annotation type or None if found none or more than one."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn annotation comment or None if found none or more than one.", "response": "def get_annotation_comment(self, r_term):\n        \"\"\"Returns annotation comment or None if found none or more than one.\n        Reports errors.\n        \"\"\"\n        comment_list = list(self.graph.triples((r_term, RDFS.comment, None)))\n        if len(comment_list) > 1:\n            self.error = True\n            msg = 'Annotation can have at most one comment.'\n            self.logger.log(msg)\n            return\n        else:\n            return six.text_type(comment_list[0][2])"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nreturns annotation date or None if not found.", "response": "def get_annotation_date(self, r_term):\n        \"\"\"Returns annotation date or None if not found.\n        Reports error on failure.\n        Note does not check value format.\n        \"\"\"\n        annotation_date_list = list(self.graph.triples((r_term, self.spdx_namespace['annotationDate'], None)))\n        if len(annotation_date_list) != 1:\n            self.error = True\n            msg = 'Annotation must have exactly one annotation date.'\n            self.logger.log(msg)\n            return\n        return six.text_type(annotation_date_list[0][2])"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef parse(self, fil):\n        self.error = False\n        self.graph = Graph()\n        self.graph.parse(file=fil, format='xml')\n        self.doc = document.Document()\n\n        for s, _p, o in self.graph.triples((None, RDF.type, self.spdx_namespace['SpdxDocument'])):\n            self.parse_doc_fields(s)\n\n        for s, _p, o in self.graph.triples((None, RDF.type, self.spdx_namespace['ExternalDocumentRef'])):\n            self.parse_ext_doc_ref(s)\n\n        for s, _p, o in self.graph.triples((None, RDF.type, self.spdx_namespace['CreationInfo'])):\n            self.parse_creation_info(s)\n\n        for s, _p, o in self.graph.triples((None, RDF.type, self.spdx_namespace['Package'])):\n            self.parse_package(s)\n\n        for s, _p, o in self.graph.triples((None, self.spdx_namespace['referencesFile'], None)):\n            self.parse_file(o)\n\n        for s, _p, o in self.graph.triples((None, self.spdx_namespace['reviewed'], None)):\n            self.parse_review(o)\n\n        for s, _p, o in self.graph.triples((None, self.spdx_namespace['annotation'], None)):\n            self.parse_annotation(o)\n\n        validation_messages = []\n        # Report extra errors if self.error is False otherwise there will be\n        # redundent messages\n        validation_messages = self.doc.validate(validation_messages)\n        if not self.error:\n            if validation_messages:\n                for msg in validation_messages:\n                    self.logger.log(msg)\n                self.error = True\n        return self.doc, self.error", "response": "Parses a file and returns a document object."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nparses creation info from a term.", "response": "def parse_creation_info(self, ci_term):\n        \"\"\"\n        Parse creators, created and comment.\n        \"\"\"\n        for _s, _p, o in self.graph.triples((ci_term, self.spdx_namespace['creator'], None)):\n            try:\n                ent = self.builder.create_entity(self.doc, six.text_type(o))\n                self.builder.add_creator(self.doc, ent)\n            except SPDXValueError:\n                self.value_error('CREATOR_VALUE', o)\n\n        for _s, _p, o in self.graph.triples((ci_term, self.spdx_namespace['created'], None)):\n            try:\n                self.builder.set_created_date(self.doc, six.text_type(o))\n            except SPDXValueError:\n                self.value_error('CREATED_VALUE', o)\n            except CardinalityError:\n                self.more_than_one_error('created')\n                break\n\n        for _s, _p, o in self.graph.triples((ci_term, RDFS.comment, None)):\n            try:\n                self.builder.set_creation_comment(self.doc, six.text_type(o))\n            except CardinalityError:\n                self.more_than_one_error('CreationInfo comment')\n                break\n        for _s, _p, o in self.graph.triples((ci_term, self.spdx_namespace['licenseListVersion'], None)):\n            try:\n                self.builder.set_lics_list_ver(self.doc, six.text_type(o))\n            except CardinalityError:\n                self.more_than_one_error('licenseListVersion')\n                break\n            except SPDXValueError:\n                self.value_error('LL_VALUE', o)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nparsing the version data license name SPDX Identifier namespace and comment.", "response": "def parse_doc_fields(self, doc_term):\n        \"\"\"Parses the version, data license, name, SPDX Identifier, namespace,\n        and comment.\"\"\"\n        try:\n            self.builder.set_doc_spdx_id(self.doc, doc_term)\n        except SPDXValueError:\n            self.value_error('DOC_SPDX_ID_VALUE', doc_term)\n        try:\n            if doc_term.count('#', 0, len(doc_term)) <= 1:\n                doc_namespace = doc_term.split('#')[0]\n                self.builder.set_doc_namespace(self.doc, doc_namespace)\n            else:\n                self.value_error('DOC_NAMESPACE_VALUE', doc_term)\n        except SPDXValueError:\n            self.value_error('DOC_NAMESPACE_VALUE', doc_term)\n        for _s, _p, o in self.graph.triples((doc_term, self.spdx_namespace['specVersion'], None)):\n            try:\n                self.builder.set_doc_version(self.doc, six.text_type(o))\n            except SPDXValueError:\n                self.value_error('DOC_VERS_VALUE', o)\n            except CardinalityError:\n                self.more_than_one_error('specVersion')\n                break\n        for _s, _p, o in self.graph.triples((doc_term, self.spdx_namespace['dataLicense'], None)):\n            try:\n                self.builder.set_doc_data_lic(self.doc, six.text_type(o))\n            except SPDXValueError:\n                self.value_error('DOC_D_LICS', o)\n            except CardinalityError:\n                self.more_than_one_error('dataLicense')\n                break\n        for _s, _p, o in self.graph.triples(\n                (doc_term, self.spdx_namespace['name'], None)):\n            try:\n                self.builder.set_doc_name(self.doc, six.text_type(o))\n            except CardinalityError:\n                self.more_than_one_error('name')\n                break\n        for _s, _p, o in self.graph.triples((doc_term, RDFS.comment, None)):\n            try:\n                self.builder.set_doc_comment(self.doc, six.text_type(o))\n            except CardinalityError:\n                self.more_than_one_error('Document comment')\n                break"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef parse_ext_doc_ref(self, ext_doc_ref_term):\n        for _s, _p, o in self.graph.triples(\n                (ext_doc_ref_term,\n                 self.spdx_namespace['externalDocumentId'],\n                 None)):\n            try:\n                self.builder.set_ext_doc_id(self.doc, six.text_type(o))\n            except SPDXValueError:\n                self.value_error('EXT_DOC_REF_VALUE', 'External Document ID')\n                break\n\n        for _s, _p, o in self.graph.triples(\n                (ext_doc_ref_term,\n                 self.spdx_namespace['spdxDocument'],\n                 None)):\n            try:\n                self.builder.set_spdx_doc_uri(self.doc, six.text_type(o))\n            except SPDXValueError:\n                self.value_error('EXT_DOC_REF_VALUE', 'SPDX Document URI')\n                break\n\n        for _s, _p, checksum in self.graph.triples(\n                (ext_doc_ref_term, self.spdx_namespace['checksum'], None)):\n            for _, _, value in self.graph.triples(\n                    (checksum, self.spdx_namespace['checksumValue'], None)):\n                try:\n                    self.builder.set_chksum(self.doc, six.text_type(value))\n                except SPDXValueError:\n                    self.value_error('EXT_DOC_REF_VALUE', 'Checksum')\n                    break", "response": "Parses the external document reference term and sets the attributes of the external document object."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nvalidating the package fields.", "response": "def validate(self, messages):\n        \"\"\"\n        Validate the package fields.\n        Append user friendly error messages to the `messages` list.\n        \"\"\"\n        messages = self.validate_checksum(messages)\n        messages = self.validate_optional_str_fields(messages)\n        messages = self.validate_mandatory_str_fields(messages)\n        messages = self.validate_files(messages)\n        messages = self.validate_mandatory_fields(messages)\n        messages = self.validate_optional_fields(messages)\n\n        return messages"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nvalidating that the fields marked as optional and of type string in class class docstring are of type string in class class docstring.", "response": "def validate_optional_str_fields(self, messages):\n        \"\"\"Fields marked as optional and of type string in class\n        docstring must be of a type that provides __str__ method.\n        \"\"\"\n        FIELDS = [\n            'file_name',\n            'version',\n            'homepage',\n            'source_info',\n            'summary',\n            'description'\n        ]\n        messages = self.validate_str_fields(FIELDS, True, messages)\n\n        return messages"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nvalidates that all mandatory fields in class are of type string in class docstring.", "response": "def validate_mandatory_str_fields(self, messages):\n        \"\"\"Fields marked as Mandatory and of type string in class\n        docstring must be of a type that provides __str__ method.\n        \"\"\"\n        FIELDS = ['name', 'download_location', 'verif_code', 'cr_text']\n        messages = self.validate_str_fields(FIELDS, False, messages)\n\n        return messages"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef validate_str_fields(self, fields, optional, messages):\n        for field_str in fields:\n            field = getattr(self, field_str)\n            if field is not None:\n                # FIXME: this does not make sense???\n                attr = getattr(field, '__str__', None)\n                if not callable(attr):\n                    messages = messages + [\n                        '{0} must provide __str__ method.'.format(field)\n                    ]\n                    # Continue checking.\n            elif not optional:\n                messages = messages + [\n                    'Package {0} can not be None.'.format(field_str)\n                ]\n\n        return messages", "response": "Helper for validate_mandatory_str_field and validate_optional_str_fields"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset the document data license.", "response": "def set_doc_data_lic(self, doc, res):\n        \"\"\"\n        Set the document data license.\n        Raise exceptions:\n        - SPDXValueError if malformed value,\n        - CardinalityError if already defined.\n        \"\"\"\n        if not self.doc_data_lics_set:\n            self.doc_data_lics_set = True\n            # TODO: what is this split?\n            res_parts = res.split('/')\n            if len(res_parts) != 0:\n                identifier = res_parts[-1]\n                doc.data_license = document.License.from_identifier(identifier)\n            else:\n                raise SPDXValueError('Document::License')\n        else:\n            raise CardinalityError('Document::License')"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nset the comment of the document. Raises CardinalityError if comment already set.", "response": "def set_doc_comment(self, doc, comment):\n        \"\"\"Sets document comment, Raises CardinalityError if\n        comment already set.\n        \"\"\"\n        if not self.doc_comment_set:\n            self.doc_comment_set = True\n            doc.comment = comment\n        else:\n            raise CardinalityError('Document::Comment')"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nset the check sum of the external document reference if not already set.", "response": "def set_chksum(self, doc, chk_sum):\n        \"\"\"\n        Sets the external document reference's check sum, if not already set.\n        chk_sum - The checksum value in the form of a string.\n        \"\"\"\n        if chk_sum:\n            doc.ext_document_references[-1].check_sum = checksum.Algorithm(\n                'SHA1', chk_sum)\n        else:\n            raise SPDXValueError('ExternalDocumentRef::Checksum')"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting the creation comment for the current object.", "response": "def set_creation_comment(self, doc, comment):\n        \"\"\"Sets creation comment, Raises CardinalityError if\n        comment already set.\n        Raises SPDXValueError if not free form text.\n        \"\"\"\n        if not self.creation_comment_set:\n            self.creation_comment_set = True\n            doc.creation_info.comment = comment\n            return True\n        else:\n            raise CardinalityError('CreationInfo::Comment')"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_pkg_chk_sum(self, doc, chk_sum):\n        self.assert_package_exists()\n        if not self.package_chk_sum_set:\n            self.package_chk_sum_set = True\n            doc.package.check_sum = checksum.Algorithm('SHA1', chk_sum)\n        else:\n            raise CardinalityError('Package::CheckSum')", "response": "Sets the package check sum."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the package s source information if not already set.", "response": "def set_pkg_source_info(self, doc, text):\n        \"\"\"Sets the package's source information, if not already set.\n        text - Free form text.\n        Raises CardinalityError if already defined.\n        Raises OrderError if no package previously defined.\n        \"\"\"\n        self.assert_package_exists()\n        if not self.package_source_info_set:\n            self.package_source_info_set = True\n            doc.package.source_info = text\n            return True\n        else:\n            raise CardinalityError('Package::SourceInfo')"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef set_pkg_verif_code(self, doc, code):\n        self.assert_package_exists()\n        if not self.package_verif_set:\n            self.package_verif_set = True\n            doc.package.verif_code = code\n        else:\n            raise CardinalityError('Package::VerificationCode')", "response": "Sets the verification code for the given package."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef set_pkg_excl_file(self, doc, filename):\n        self.assert_package_exists()\n        doc.package.add_exc_file(filename)", "response": "Sets the verification code excluded file. Raises OrderError if no package was defined."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_pkg_license_comment(self, doc, text):\n        self.assert_package_exists()\n        if not self.package_license_comment_set:\n            self.package_license_comment_set = True\n            doc.package.license_comment = text\n            return True\n        else:\n            raise CardinalityError('Package::LicenseComment')", "response": "Sets the package s license comment."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef set_pkg_cr_text(self, doc, text):\n        self.assert_package_exists()\n        if not self.package_cr_text_set:\n            self.package_cr_text_set = True\n            doc.package.cr_text = text\n        else:\n            raise CardinalityError('Package::CopyrightText')", "response": "Sets the package s license comment text."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef set_pkg_summary(self, doc, text):\n        self.assert_package_exists()\n        if not self.package_summary_set:\n            self.package_summary_set = True\n            doc.package.summary = text\n        else:\n            raise CardinalityError('Package::Summary')", "response": "Set the package summary."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nsets the package s description.", "response": "def set_pkg_desc(self, doc, text):\n        \"\"\"Set's the package's description.\n        Raises CardinalityError if description already set.\n        Raises OrderError if no package previously defined.\n        \"\"\"\n        self.assert_package_exists()\n        if not self.package_desc_set:\n            self.package_desc_set = True\n            doc.package.description = text\n        else:\n            raise CardinalityError('Package::Description')"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nsets the file check sum for the given document.", "response": "def set_file_chksum(self, doc, chk_sum):\n        \"\"\"Sets the file check sum, if not already set.\n        chk_sum - A string\n        Raises CardinalityError if already defined.\n        Raises OrderError if no package previously defined.\n        \"\"\"\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_chksum_set:\n                self.file_chksum_set = True\n                self.file(doc).chk_sum = checksum.Algorithm('SHA1', chk_sum)\n                return True\n            else:\n                raise CardinalityError('File::CheckSum')\n        else:\n            raise OrderError('File::CheckSum')"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef set_file_license_comment(self, doc, text):\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_license_comment_set:\n                self.file_license_comment_set = True\n                self.file(doc).license_comment = text\n                return True\n            else:\n                raise CardinalityError('File::LicenseComment')\n        else:\n            raise OrderError('File::LicenseComment')", "response": "Sets the license comment for the file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting copyright text for a file in the order.", "response": "def set_file_copyright(self, doc, text):\n        \"\"\"Raises OrderError if no package or file defined.\n        Raises CardinalityError if more than one.\n        \"\"\"\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_copytext_set:\n                self.file_copytext_set = True\n                self.file(doc).copyright = text\n                return True\n            else:\n                raise CardinalityError('File::CopyRight')\n        else:\n            raise OrderError('File::CopyRight')"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef set_file_comment(self, doc, text):\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_comment_set:\n                self.file_comment_set = True\n                self.file(doc).comment = text\n                return True\n            else:\n                raise CardinalityError('File::Comment')\n        else:\n            raise OrderError('File::Comment')", "response": "Sets the comment for the file."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef set_file_notice(self, doc, text):\n        if self.has_package(doc) and self.has_file(doc):\n            if not self.file_notice_set:\n                self.file_notice_set = True\n                self.file(doc).notice = tagvaluebuilders.str_from_text(text)\n                return True\n            else:\n                raise CardinalityError('File::Notice')\n        else:\n            raise OrderError('File::Notice')", "response": "Sets the notice field in the file object."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef add_annotation_type(self, doc, annotation_type):\n        if len(doc.annotations) != 0:\n            if not self.annotation_type_set:\n                if annotation_type.endswith('annotationType_other'):\n                    self.annotation_type_set = True\n                    doc.annotations[-1].annotation_type = 'OTHER'\n                    return True\n                elif annotation_type.endswith('annotationType_review'):\n                    self.annotation_type_set = True\n                    doc.annotations[-1].annotation_type = 'REVIEW'\n                    return True\n                else:\n                    raise SPDXValueError('Annotation::AnnotationType')\n            else:\n                raise CardinalityError('Annotation::AnnotationType')\n        else:\n            raise OrderError('Annotation::AnnotationType')", "response": "Sets the annotation type. Raises CardinalityError if no annotator defined before."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef validate(self, messages):\n        messages = self.validate_version(messages)\n        messages = self.validate_data_lics(messages)\n        messages = self.validate_name(messages)\n        messages = self.validate_spdx_id(messages)\n        messages = self.validate_namespace(messages)\n        messages = self.validate_ext_document_references(messages)\n        messages = self.validate_creation_info(messages)\n        messages = self.validate_package(messages)\n        messages = self.validate_extracted_licenses(messages)\n        messages = self.validate_reviews(messages)\n\n        return messages", "response": "Validate all fields of the document and update the list with user friendly error messages for display."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef include(f):\n  '''\n  includes the contents of a file on disk.\n  takes a filename\n  '''\n  fl = open(f, 'r')\n  data = fl.read()\n  fl.close()\n  return raw(data)", "response": "includes the contents of a file on disk."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\npipes the output of a program", "response": "def system(cmd, data=None):\n  '''\n  pipes the output of a program\n  '''\n  import subprocess\n  s = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE)\n  out, err = s.communicate(data)\n  return out.decode('utf8')"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef escape(data, quote=True):  # stoled from std lib cgi\n  '''\n  Escapes special characters into their html entities\n  Replace special characters \"&\", \"<\" and \">\" to HTML-safe sequences.\n  If the optional flag quote is true, the quotation mark character (\")\n  is also translated.\n\n  This is used to escape content that appears in the body of an HTML cocument\n  '''\n  data = data.replace(\"&\", \"&amp;\")  # Must be done first!\n  data = data.replace(\"<\", \"&lt;\")\n  data = data.replace(\">\", \"&gt;\")\n  if quote:\n    data = data.replace('\"', \"&quot;\")\n  return data", "response": "Escapes special characters into HTML - safe sequences."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef unescape(data):\n  '''\n  unescapes html entities. the opposite of escape.\n  '''\n  cc = re.compile(r'&(?:(?:#(\\d+))|([^;]+));')\n\n  result = []\n  m = cc.search(data)\n  while m:\n    result.append(data[0:m.start()])\n    d = m.group(1)\n    if d:\n      d = int(d)\n      result.append(unichr(d))\n    else:\n      d = _unescape.get(m.group(2), ord('?'))\n      result.append(unichr(d))\n\n    data = data[m.end():]\n    m = cc.search(data)\n\n  result.append(data)\n  return ''.join(result)", "response": "unescapes html entities. the opposite of escape.\n \u00ab unescapes html entities. the opposite of escape.\n \u00bb"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsetting attributes on the current active tag context ethernet.", "response": "def attr(*args, **kwargs):\n  '''\n  Set attributes on the current active tag context\n  '''\n  ctx = dom_tag._with_contexts[_get_thread_context()]\n  if ctx and ctx[-1]:\n    dicts = args + (kwargs,)\n    for d in dicts:\n      for attr, value in d.items():\n        ctx[-1].tag.set_attribute(*dom_tag.clean_pair(attr, value))\n  else:\n    raise ValueError('not in a tag context')"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_attribute(self, key, value):\n    '''\n    Add or update the value of an attribute.\n    '''\n    if isinstance(key, int):\n      self.children[key] = value\n    elif isinstance(key, basestring):\n      self.attributes[key] = value\n    else:\n      raise TypeError('Only integer and string types are valid for assigning '\n          'child tags and attributes, respectively.')", "response": "Add or update the value of an attribute."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsetting the document of the node.", "response": "def setdocument(self, doc):\n    '''\n    Creates a reference to the parent document to allow for partial-tree\n    validation.\n    '''\n    # assume that a document is correct in the subtree\n    if self.document != doc:\n      self.document = doc\n      for i in self.children:\n        if not isinstance(i, dom_tag): return\n        i.setdocument(doc)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds new child tags.", "response": "def add(self, *args):\n    '''\n    Add new child tags.\n    '''\n    for obj in args:\n      if isinstance(obj, numbers.Number):\n        # Convert to string so we fall into next if block\n        obj = str(obj)\n\n      if isinstance(obj, basestring):\n        obj = escape(obj)\n        self.children.append(obj)\n\n      elif isinstance(obj, dom_tag):\n        ctx = dom_tag._with_contexts[_get_thread_context()]\n        if ctx and ctx[-1]:\n          ctx[-1].used.add(obj)\n        self.children.append(obj)\n        obj.parent = self\n        obj.setdocument(self.document)\n\n      elif isinstance(obj, dict):\n        for attr, value in obj.items():\n          self.set_attribute(*dom_tag.clean_pair(attr, value))\n\n      elif hasattr(obj, '__iter__'):\n        for subobj in obj:\n          self.add(subobj)\n\n      else:  # wtf is it?\n        raise ValueError('%r not a tag or string.' % obj)\n\n    if len(args) == 1:\n      return args[0]\n\n    return args"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nclean the attribute name for shorthand and work arounds for limitations in Python s syntax", "response": "def clean_attribute(attribute):\n    '''\n    Normalize attribute names for shorthand and work arounds for limitations\n    in Python's syntax\n    '''\n\n    # Shorthand\n    attribute = {\n      'cls': 'class',\n      'className': 'class',\n      'class_name': 'class',\n      'fr': 'for',\n      'html_for': 'for',\n      'htmlFor': 'for',\n    }.get(attribute, attribute)\n\n    # Workaround for Python's reserved words\n    if attribute[0] == '_':\n      attribute = attribute[1:]\n\n    # Workaround for dash\n    if attribute in set(['http_equiv']) or attribute.startswith('data_'):\n      attribute = attribute.replace('_', '-').lower()\n\n    # Workaround for colon\n    if attribute.split('_')[0] in ('xlink', 'xml', 'xmlns'):\n      attribute = attribute.replace('_', ':', 1).lower()\n\n    return attribute"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef clean_pair(cls, attribute, value):\n    '''\n    This will call `clean_attribute` on the attribute and also allows for the\n    creation of boolean attributes.\n\n    Ex. input(selected=True) is equivalent to input(selected=\"selected\")\n    '''\n    attribute = cls.clean_attribute(attribute)\n\n    # Check for boolean attributes\n    # (i.e. selected=True becomes selected=\"selected\")\n    if value is True:\n      value = attribute\n\n    if value is False:\n      value = \"false\"\n\n    return (attribute, value)", "response": "This method will call clean_attribute on the attribute and also check for boolean attributes."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates a new empty tag if not present and renders the DOCTYPE and tag tree.", "response": "def render(self, *args, **kwargs):\n    '''\n    Creates a <title> tag if not present and renders the DOCTYPE and tag tree.\n    '''\n    r = []\n\n    #Validates the tag tree and adds the doctype if one was set\n    if self.doctype:\n      r.append(self.doctype)\n      r.append('\\n')\n    r.append(super(document, self).render(*args, **kwargs))\n\n    return u''.join(r)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget element with matching id value.", "response": "def getElementById(self, id):\n    '''\n    DOM API: Returns single element with matching id value.\n    '''\n    results = self.get(id=id)\n    if len(results) > 1:\n      raise ValueError('Multiple tags with id \"%s\".' % id)\n    elif results:\n      return results[0]\n    else:\n      return None"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef getElementsByTagName(self, name):\n    '''\n    DOM API: Returns all tags that match name.\n    '''\n    if isinstance(name, basestring):\n      return self.get(name.lower())\n    else:\n      return None", "response": "getElementsByTagName - Returns all tags that match name."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef start(self):\n        self.outgoing_q = zmq_pipes.TasksOutgoing(\n            \"127.0.0.1\", self.interchange_port_range)\n        self.incoming_q = zmq_pipes.ResultsIncoming(\n            \"127.0.0.1\", self.interchange_port_range)\n\n        self.is_alive = True\n\n        self._queue_management_thread = None\n        self._start_queue_management_thread()\n        self._start_local_queue_process()\n\n        logger.debug(\"Created management thread: {}\"\n                     .format(self._queue_management_thread))\n\n        if self.provider:\n            # debug_opts = \"--debug\" if self.worker_debug else \"\"\n            l_cmd = self.launch_cmd.format(  # debug=debug_opts,\n                task_url=self.worker_task_url,\n                workers_per_node=self.workers_per_node,\n                logdir=\"{}/{}\".format(self.run_dir, self.label))\n            self.launch_cmd = l_cmd\n            logger.debug(\"Launch command: {}\".format(self.launch_cmd))\n\n            self._scaling_enabled = self.provider.scaling_enabled\n            logger.debug(\n                \"Starting LowLatencyExecutor with provider:\\n%s\", self.provider)\n            if hasattr(self.provider, 'init_blocks'):\n                try:\n                    for i in range(self.provider.init_blocks):\n                        block = self.provider.submit(\n                            self.launch_cmd, 1, self.workers_per_node)\n                        logger.debug(\"Launched block {}:{}\".format(i, block))\n                        if not block:\n                            raise(ScalingFailed(self.provider.label,\n                                                \"Attempts to provision nodes via provider has failed\"))\n                        self.blocks.extend([block])\n\n                except Exception as e:\n                    logger.error(\"Scaling out failed: {}\".format(e))\n                    raise e\n        else:\n            self._scaling_enabled = False\n            logger.debug(\"Starting LowLatencyExecutor with no provider\")", "response": "Start the Interchange process and connect to it."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nstarting the local queue process.", "response": "def _start_local_queue_process(self):\n        \"\"\" TODO: docstring \"\"\"\n\n        comm_q = Queue(maxsize=10)\n        self.queue_proc = Process(target=interchange.starter,\n                                  args=(comm_q,),\n                                  kwargs={\"client_ports\": (self.outgoing_q.port,\n                                                           self.incoming_q.port),\n                                          \"worker_port\": self.worker_port,\n                                          \"worker_port_range\": self.worker_port_range\n                                          # TODO: logdir and logging level\n                                          })\n        self.queue_proc.start()\n\n        try:\n            worker_port = comm_q.get(block=True, timeout=120)\n            logger.debug(\n                \"Got worker port {} from interchange\".format(worker_port))\n        except queue.Empty:\n            logger.error(\n                \"Interchange has not completed initialization in 120s. Aborting\")\n            raise Exception(\"Interchange failed to start\")\n\n        self.worker_task_url = \"tcp://{}:{}\".format(\n            self.address, worker_port)"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _start_queue_management_thread(self):\n        if self._queue_management_thread is None:\n            logger.debug(\"Starting queue management thread\")\n            self._queue_management_thread = threading.Thread(\n                target=self._queue_management_worker)\n            self._queue_management_thread.daemon = True\n            self._queue_management_thread.start()\n            logger.debug(\"Started queue management thread\")\n\n        else:\n            logger.debug(\"Management thread already exists, returning\")", "response": "Start the queue management thread."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef _queue_management_worker(self):\n        logger.debug(\"[MTHREAD] queue management worker starting\")\n\n        while True:\n            task_id, buf = self.incoming_q.get()  # TODO: why does this hang?\n            msg = deserialize_object(buf)[0]\n            # TODO: handle exceptions\n            task_fut = self.tasks[task_id]\n            logger.debug(\"Got response for task id {}\".format(task_id))\n\n            if \"result\" in msg:\n                task_fut.set_result(msg[\"result\"])\n\n            elif \"exception\" in msg:\n                # TODO: handle exception\n                pass\n            elif 'exception' in msg:\n                logger.warning(\"Task: {} has returned with an exception\")\n                try:\n                    s, _ = deserialize_object(msg['exception'])\n                    exception = ValueError(\"Remote exception description: {}\".format(s))\n                    task_fut.set_exception(exception)\n                except Exception as e:\n                    # TODO could be a proper wrapped exception?\n                    task_fut.set_exception(\n                        DeserializationError(\"Received exception, but handling also threw an exception: {}\".format(e)))\n\n            else:\n                raise BadMessage(\n                    \"Message received is neither result nor exception\")\n\n            if not self.is_alive:\n                break\n\n        logger.info(\"[MTHREAD] queue management worker finished\")", "response": "This function is the main thread that handles the queue management."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef scale_in(self, blocks):\n        to_kill = self.blocks[:blocks]\n        if self.provider:\n            r = self.provider.cancel(to_kill)\n        return r", "response": "Scale in the number of active blocks by specified amount."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a JSON message to identify the worker to the interchange", "response": "def create_reg_message(self):\n        \"\"\" Creates a registration message to identify the worker to the interchange\n        \"\"\"\n        msg = {'parsl_v': PARSL_VERSION,\n               'python_v': \"{}.{}.{}\".format(sys.version_info.major,\n                                             sys.version_info.minor,\n                                             sys.version_info.micro),\n               'os': platform.system(),\n               'hname': platform.node(),\n               'dir': os.getcwd(),\n        }\n        b_msg = json.dumps(msg).encode('utf-8')\n        return b_msg"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nsends a heartbeat to the incoming task queue", "response": "def heartbeat(self):\n        \"\"\" Send heartbeat to the incoming task queue\n        \"\"\"\n        heartbeat = (HEARTBEAT_CODE).to_bytes(4, \"little\")\n        r = self.task_incoming.send(heartbeat)\n        logger.debug(\"Return from heartbeat : {}\".format(r))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreceive a result from the workers and sends it out via 0mq", "response": "def recv_result_from_workers(self):\n        \"\"\" Receives a results from the MPI worker pool and send it out via 0mq\n\n        Returns:\n        --------\n            result: task result from the workers\n        \"\"\"\n        info = MPI.Status()\n        result = self.comm.recv(source=MPI.ANY_SOURCE, tag=RESULT_TAG, status=info)\n        logger.debug(\"Received result from workers: {}\".format(result))\n        return result"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef recv_task_request_from_workers(self):\n        info = MPI.Status()\n        comm.recv(source=MPI.ANY_SOURCE, tag=TASK_REQUEST_TAG, status=info)\n        worker_rank = info.Get_source()\n        logger.info(\"Received task request from worker:{}\".format(worker_rank))\n        return worker_rank", "response": "Receives 1 task request from MPI comm\n\n        Returns the worker_rank of the task request."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\npulling tasks from the incoming queue onto the internal task queue.", "response": "def pull_tasks(self, kill_event):\n        \"\"\" Pulls tasks from the incoming tasks 0mq pipe onto the internal\n        pending task queue\n\n        Parameters:\n        -----------\n        kill_event : threading.Event\n              Event to let the thread know when it is time to die.\n        \"\"\"\n        logger.info(\"[TASK PULL THREAD] starting\")\n        poller = zmq.Poller()\n        poller.register(self.task_incoming, zmq.POLLIN)\n\n        # Send a registration message\n        msg = self.create_reg_message()\n        logger.debug(\"Sending registration message: {}\".format(msg))\n        self.task_incoming.send(msg)\n        last_beat = time.time()\n        last_interchange_contact = time.time()\n        task_recv_counter = 0\n\n        poll_timer = 1\n\n        while not kill_event.is_set():\n            time.sleep(LOOP_SLOWDOWN)\n            ready_worker_count = self.ready_worker_queue.qsize()\n            pending_task_count = self.pending_task_queue.qsize()\n\n            logger.debug(\"[TASK_PULL_THREAD] ready workers:{}, pending tasks:{}\".format(ready_worker_count,\n                                                                                        pending_task_count))\n\n            if time.time() > last_beat + self.heartbeat_period:\n                self.heartbeat()\n                last_beat = time.time()\n\n            if pending_task_count < self.max_queue_size and ready_worker_count > 0:\n                logger.debug(\"[TASK_PULL_THREAD] Requesting tasks: {}\".format(ready_worker_count))\n                msg = ((ready_worker_count).to_bytes(4, \"little\"))\n                self.task_incoming.send(msg)\n\n            socks = dict(poller.poll(timeout=poll_timer))\n\n            if self.task_incoming in socks and socks[self.task_incoming] == zmq.POLLIN:\n                _, pkl_msg = self.task_incoming.recv_multipart()\n                tasks = pickle.loads(pkl_msg)\n                last_interchange_contact = time.time()\n\n                if tasks == 'STOP':\n                    logger.critical(\"[TASK_PULL_THREAD] Received stop request\")\n                    kill_event.set()\n                    break\n\n                elif tasks == HEARTBEAT_CODE:\n                    logger.debug(\"Got heartbeat from interchange\")\n\n                else:\n                    # Reset timer on receiving message\n                    poll_timer = 1\n                    task_recv_counter += len(tasks)\n                    logger.debug(\"[TASK_PULL_THREAD] Got tasks: {} of {}\".format([t['task_id'] for t in tasks],\n                                                                                 task_recv_counter))\n                    for task in tasks:\n                        self.pending_task_queue.put(task)\n            else:\n                logger.debug(\"[TASK_PULL_THREAD] No incoming tasks\")\n                # Limit poll duration to heartbeat_period\n                # heartbeat_period is in s vs poll_timer in ms\n                poll_timer = min(self.heartbeat_period * 1000, poll_timer * 2)\n\n                # Only check if no messages were received.\n                if time.time() > last_interchange_contact + self.heartbeat_threshold:\n                    logger.critical(\"[TASK_PULL_THREAD] Missing contact with interchange beyond heartbeat_threshold\")\n                    kill_event.set()\n                    logger.critical(\"[TASK_PULL_THREAD] Exiting\")\n                    break"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef push_results(self, kill_event):\n\n        # We set this timeout so that the thread checks the kill_event and does not\n        # block forever on the internal result queue\n        timeout = 0.1\n        # timer = time.time()\n        logger.debug(\"[RESULT_PUSH_THREAD] Starting thread\")\n\n        while not kill_event.is_set():\n            time.sleep(LOOP_SLOWDOWN)\n            try:\n                items = []\n                while not self.pending_result_queue.empty():\n                    r = self.pending_result_queue.get(block=True)\n                    items.append(r)\n                if items:\n                    self.result_outgoing.send_multipart(items)\n\n            except queue.Empty:\n                logger.debug(\"[RESULT_PUSH_THREAD] No results to send in past {}seconds\".format(timeout))\n\n            except Exception as e:\n                logger.exception(\"[RESULT_PUSH_THREAD] Got an exception : {}\".format(e))\n\n        logger.critical(\"[RESULT_PUSH_THREAD] Exiting\")", "response": "Sends out results to the result queue."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nstart the Manager process.", "response": "def start(self):\n        \"\"\" Start the Manager process.\n\n        The worker loops on this:\n\n        1. If the last message sent was older than heartbeat period we send a heartbeat\n        2.\n\n\n        TODO: Move task receiving to a thread\n        \"\"\"\n\n        self.comm.Barrier()\n        logger.debug(\"Manager synced with workers\")\n\n        self._kill_event = threading.Event()\n        self._task_puller_thread = threading.Thread(target=self.pull_tasks,\n                                                    args=(self._kill_event,))\n        self._result_pusher_thread = threading.Thread(target=self.push_results,\n                                                      args=(self._kill_event,))\n        self._task_puller_thread.start()\n        self._result_pusher_thread.start()\n\n        start = None\n\n        result_counter = 0\n        task_recv_counter = 0\n        task_sent_counter = 0\n\n        logger.info(\"Loop start\")\n        while not self._kill_event.is_set():\n            time.sleep(LOOP_SLOWDOWN)\n\n            # In this block we attempt to probe MPI for a set amount of time,\n            # and if we have exhausted all available MPI events, we move on\n            # to the next block. The timer and counter trigger balance\n            # fairness and responsiveness.\n            timer = time.time() + 0.05\n            counter = min(10, comm.size)\n            while time.time() < timer:\n                info = MPI.Status()\n\n                if counter > 10:\n                    logger.debug(\"Hit max mpi events per round\")\n                    break\n\n                if not self.comm.Iprobe(status=info):\n                    logger.debug(\"Timer expired, processed {} mpi events\".format(counter))\n                    break\n                else:\n                    tag = info.Get_tag()\n                    logger.info(\"Message with tag {} received\".format(tag))\n\n                    counter += 1\n                    if tag == RESULT_TAG:\n                        result = self.recv_result_from_workers()\n                        self.pending_result_queue.put(result)\n                        result_counter += 1\n\n                    elif tag == TASK_REQUEST_TAG:\n                        worker_rank = self.recv_task_request_from_workers()\n                        self.ready_worker_queue.put(worker_rank)\n\n                    else:\n                        logger.error(\"Unknown tag {} - ignoring this message and continuing\".format(tag))\n\n            available_worker_cnt = self.ready_worker_queue.qsize()\n            available_task_cnt = self.pending_task_queue.qsize()\n            logger.debug(\"[MAIN] Ready workers: {} Ready tasks: {}\".format(available_worker_cnt,\n                                                                           available_task_cnt))\n            this_round = min(available_worker_cnt, available_task_cnt)\n            for i in range(this_round):\n                worker_rank = self.ready_worker_queue.get()\n                task = self.pending_task_queue.get()\n                comm.send(task, dest=worker_rank, tag=worker_rank)\n                task_sent_counter += 1\n                logger.debug(\"Assigning worker:{} task:{}\".format(worker_rank, task['task_id']))\n\n            if not start:\n                start = time.time()\n\n            logger.debug(\"Tasks recvd:{} Tasks dispatched:{} Results recvd:{}\".format(\n                task_recv_counter, task_sent_counter, result_counter))\n            # print(\"[{}] Received: {}\".format(self.identity, msg))\n            # time.sleep(random.randint(4,10)/10)\n\n        self._task_puller_thread.join()\n        self._result_pusher_thread.join()\n\n        self.task_incoming.close()\n        self.result_outgoing.close()\n        self.context.term()\n\n        delta = time.time() - start\n        logger.info(\"mpi_worker_pool ran for {} seconds\".format(delta))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef udp_messenger(domain_name, UDP_IP, UDP_PORT, sock_timeout, message):\n    try:\n        if message is None:\n            raise ValueError(\"message was none\")\n\n        encoded_message = bytes(message, \"utf-8\")\n\n        if encoded_message is None:\n            raise ValueError(\"utf-8 encoding of message failed\")\n\n        if domain_name:\n            try:\n                UDP_IP = socket.gethostbyname(domain_name)\n            except Exception:\n                # (False, \"Domain lookup failed, defaulting to {0}\".format(UDP_IP))\n                pass\n\n        if UDP_IP is None:\n            raise Exception(\"UDP_IP is None\")\n\n        if UDP_PORT is None:\n            raise Exception(\"UDP_PORT is None\")\n\n        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # UDP\n        sock.settimeout(sock_timeout)\n        sock.sendto(bytes(message, \"utf-8\"), (UDP_IP, UDP_PORT))\n        sock.close()\n\n    except socket.timeout:\n        logger.debug(\"Failed to send usage tracking data: socket timeout\")\n    except OSError as e:\n        logger.debug(\"Failed to send usage tracking data: OSError: {}\".format(e))\n    except Exception as e:\n        logger.debug(\"Failed to send usage tracking data: Exception: {}\".format(e))", "response": "Send a message to the UDP socket."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef check_tracking_enabled(self):\n        track = True   # By default we track usage\n        test = False  # By default we are not in testing mode\n\n        testvar = str(os.environ.get(\"PARSL_TESTING\", 'None')).lower()\n        if testvar == 'true':\n            test = True\n\n        if not self.config.usage_tracking:\n            track = False\n\n        envvar = str(os.environ.get(\"PARSL_TRACKING\", True)).lower()\n        if envvar == \"false\":\n            track = False\n\n        return test, track", "response": "Returns True if tracking is enabled False if tracking is disabled"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nconstruct a json string that contains the start of the DFK.", "response": "def construct_start_message(self):\n        \"\"\"Collect preliminary run info at the start of the DFK.\n\n        Returns :\n              - Message dict dumped as json string, ready for UDP\n        \"\"\"\n        uname = getpass.getuser().encode('latin1')\n        hashed_username = hashlib.sha256(uname).hexdigest()[0:10]\n        hname = socket.gethostname().encode('latin1')\n        hashed_hostname = hashlib.sha256(hname).hexdigest()[0:10]\n        message = {'uuid': self.uuid,\n                   'uname': hashed_username,\n                   'hname': hashed_hostname,\n                   'test': self.test_mode,\n                   'parsl_v': self.parsl_version,\n                   'python_v': self.python_version,\n                   'os': platform.system(),\n                   'os_v': platform.release(),\n                   'start': time.time()}\n\n        return json.dumps(message)"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconstructs the final run information at the time of DFK cleanup.", "response": "def construct_end_message(self):\n        \"\"\"Collect the final run information at the time of DFK cleanup.\n\n        Returns:\n             - Message dict dumped as json string, ready for UDP\n        \"\"\"\n        app_count = self.dfk.task_count\n\n        site_count = len([x for x in self.dfk.config.executors if x.managed])\n\n        app_fails = len([t for t in self.dfk.tasks if\n                         self.dfk.tasks[t]['status'] in FINAL_FAILURE_STATES])\n\n        message = {'uuid': self.uuid,\n                   'end': time.time(),\n                   't_apps': app_count,\n                   'sites': site_count,\n                   'c_time': None,\n                   'failed': app_fails,\n                   'test': self.test_mode,\n                   }\n\n        return json.dumps(message)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nsending a UDP message.", "response": "def send_UDP_message(self, message):\n        \"\"\"Send UDP message.\"\"\"\n        x = 0\n        if self.tracking_enabled:\n            try:\n                proc = udp_messenger(self.domain_name, self.UDP_IP, self.UDP_PORT, self.sock_timeout, message)\n                self.procs.append(proc)\n            except Exception as e:\n                logger.debug(\"Usage tracking failed: {}\".format(e))\n        else:\n            x = -1\n\n        return x"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef send_message(self):\n        start = time.time()\n        message = None\n        if not self.initialized:\n            message = self.construct_start_message()\n            self.initialized = True\n        else:\n            message = self.construct_end_message()\n\n        self.send_UDP_message(message)\n        end = time.time()\n\n        return end - start", "response": "Send a message over UDP."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nadds a stream log handler to the log file.", "response": "def set_file_logger(filename: str, name: str = 'parsl', level: int = logging.DEBUG, format_string: Optional[str] = None):\n    \"\"\"Add a stream log handler.\n\n    Args:\n        - filename (string): Name of the file to write logs to\n        - name (string): Logger name\n        - level (logging.LEVEL): Set the logging level.\n        - format_string (string): Set the format string\n\n    Returns:\n       -  None\n    \"\"\"\n    if format_string is None:\n        format_string = \"%(asctime)s.%(msecs)03d %(name)s:%(lineno)d [%(levelname)s]  %(message)s\"\n\n    logger = logging.getLogger(name)\n    logger.setLevel(logging.DEBUG)\n    handler = logging.FileHandler(filename)\n    handler.setLevel(level)\n    formatter = logging.Formatter(format_string, datefmt='%Y-%m-%d %H:%M:%S')\n    handler.setFormatter(formatter)\n    logger.addHandler(handler)\n\n    # see note in set_stream_logger for notes about logging\n    # concurrent.futures\n    futures_logger = logging.getLogger(\"concurrent.futures\")\n    futures_logger.addHandler(handler)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef start_file_logger(filename, name='database_manager', level=logging.DEBUG, format_string=None):\n    if format_string is None:\n        format_string = \"%(asctime)s %(name)s:%(lineno)d [%(levelname)s]  %(message)s\"\n\n    global logger\n    logger = logging.getLogger(name)\n    logger.setLevel(level)\n    handler = logging.FileHandler(filename)\n    handler.setLevel(level)\n    formatter = logging.Formatter(format_string, datefmt='%Y-%m-%d %H:%M:%S')\n    handler.setFormatter(formatter)\n    logger.addHandler(handler)\n    return logger", "response": "Start a logger for a file."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nstarts the database manager process", "response": "def dbm_starter(priority_msgs, resource_msgs, *args, **kwargs):\n    \"\"\"Start the database manager process\n\n    The DFK should start this function. The args, kwargs match that of the monitoring config\n\n    \"\"\"\n    dbm = DatabaseManager(*args, **kwargs)\n    dbm.start(priority_msgs, resource_msgs)"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nstart the migration thread.", "response": "def start(self, priority_queue, resource_queue):\n\n        self._kill_event = threading.Event()\n        self._priority_queue_pull_thread = threading.Thread(target=self._migrate_logs_to_internal,\n                                                            args=(\n                                                                priority_queue, 'priority', self._kill_event,)\n                                                            )\n        self._priority_queue_pull_thread.start()\n\n        self._resource_queue_pull_thread = threading.Thread(target=self._migrate_logs_to_internal,\n                                                            args=(\n                                                                resource_queue, 'resource', self._kill_event,)\n                                                            )\n        self._resource_queue_pull_thread.start()\n\n        \"\"\"\n        maintain a set to track the tasks that are already INSERTED into database\n        to prevent race condition that the first resource message (indicate 'running' state)\n        arrives before the first task message.\n        If race condition happens, add to left_messages and operate them later\n\n        \"\"\"\n        inserted_tasks = set()\n        left_messages = {}\n\n        while (not self._kill_event.is_set() or\n               self.pending_priority_queue.qsize() != 0 or self.pending_resource_queue.qsize() != 0 or\n               priority_queue.qsize() != 0 or resource_queue.qsize() != 0):\n\n            \"\"\"\n            WORKFLOW_INFO and TASK_INFO messages\n\n            \"\"\"\n            self.logger.debug(\"\"\"Checking STOP conditions: {}, {}, {}, {}, {}\"\"\".format(\n                              self._kill_event.is_set(),\n                              self.pending_priority_queue.qsize() != 0, self.pending_resource_queue.qsize() != 0,\n                              priority_queue.qsize() != 0, resource_queue.qsize() != 0))\n\n            # This is the list of first resource messages indicating that task starts running\n            first_messages = []\n\n            # Get a batch of priority messages\n            messages = self._get_messages_in_batch(self.pending_priority_queue,\n                                                   interval=self.batching_interval,\n                                                   threshold=self.batching_threshold)\n            if messages:\n                self.logger.debug(\n                    \"Got {} messages from priority queue\".format(len(messages)))\n                update_messages, insert_messages, all_messages = [], [], []\n                for msg_type, msg in messages:\n                    if msg_type.value == MessageType.WORKFLOW_INFO.value:\n                        if \"python_version\" in msg:   # workflow start message\n                            self.logger.debug(\n                                \"Inserting workflow start info to WORKFLOW table\")\n                            self._insert(table=WORKFLOW, messages=[msg])\n                        else:                         # workflow end message\n                            self.logger.debug(\n                                \"Updating workflow end info to WORKFLOW table\")\n                            self._update(table=WORKFLOW,\n                                         columns=['run_id', 'tasks_failed_count',\n                                                  'tasks_completed_count', 'time_completed',\n                                                  'workflow_duration'],\n                                         messages=[msg])\n                    else:                             # TASK_INFO message\n                        all_messages.append(msg)\n                        if msg['task_time_returned'] is not None:\n                            update_messages.append(msg)\n                        else:\n                            inserted_tasks.add(msg['task_id'])\n                            insert_messages.append(msg)\n\n                            # check if there is an left_message for this task\n                            if msg['task_id'] in left_messages:\n                                first_messages.append(\n                                    left_messages.pop(msg['task_id']))\n\n                self.logger.debug(\n                    \"Updating and inserting TASK_INFO to all tables\")\n                self._update(table=WORKFLOW,\n                             columns=['run_id', 'tasks_failed_count',\n                                      'tasks_completed_count'],\n                             messages=update_messages)\n\n                if insert_messages:\n                    self._insert(table=TASK, messages=insert_messages)\n                    self.logger.debug(\n                        \"There are {} inserted task records\".format(len(inserted_tasks)))\n                if update_messages:\n                    self._update(table=TASK,\n                                 columns=['task_time_returned',\n                                          'task_elapsed_time', 'run_id', 'task_id'],\n                                 messages=update_messages)\n                self._insert(table=STATUS, messages=all_messages)\n\n            \"\"\"\n            RESOURCE_INFO messages\n\n            \"\"\"\n            messages = self._get_messages_in_batch(self.pending_resource_queue,\n                                                   interval=self.batching_interval,\n                                                   threshold=self.batching_threshold)\n\n            if messages or first_messages:\n                self.logger.debug(\n                    \"Got {} messages from resource queue\".format(len(messages)))\n                self._insert(table=RESOURCE, messages=messages)\n                for msg in messages:\n                    if msg['first_msg']:\n                        msg['task_status_name'] = States.running.name\n                        msg['task_time_running'] = msg['timestamp']\n                        if msg['task_id'] in inserted_tasks:\n                            first_messages.append(msg)\n                        else:\n                            left_messages[msg['task_id']] = msg\n                if first_messages:\n                    self._insert(table=STATUS, messages=first_messages)\n                    self._update(table=TASK,\n                                 columns=['task_time_running',\n                                          'run_id', 'task_id'],\n                                 messages=first_messages)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _create_task_log_info(self, task_id, fail_mode=None):\n\n        info_to_monitor = ['func_name', 'fn_hash', 'memoize', 'checkpoint', 'fail_count',\n                           'fail_history', 'status', 'id', 'time_submitted', 'time_returned', 'executor']\n\n        task_log_info = {\"task_\" + k: self.tasks[task_id][k] for k in info_to_monitor}\n        task_log_info['run_id'] = self.run_id\n        task_log_info['timestamp'] = datetime.datetime.now()\n        task_log_info['task_status_name'] = self.tasks[task_id]['status'].name\n        task_log_info['tasks_failed_count'] = self.tasks_failed_count\n        task_log_info['tasks_completed_count'] = self.tasks_completed_count\n        task_log_info['task_inputs'] = str(self.tasks[task_id]['kwargs'].get('inputs', None))\n        task_log_info['task_outputs'] = str(self.tasks[task_id]['kwargs'].get('outputs', None))\n        task_log_info['task_stdin'] = self.tasks[task_id]['kwargs'].get('stdin', None)\n        task_log_info['task_stdout'] = self.tasks[task_id]['kwargs'].get('stdout', None)\n        task_log_info['task_depends'] = None\n        if self.tasks[task_id]['depends'] is not None:\n            task_log_info['task_depends'] = \",\".join([str(t._tid) for t in self.tasks[task_id]['depends']])\n        task_log_info['task_elapsed_time'] = None\n        if self.tasks[task_id]['time_returned'] is not None:\n            task_log_info['task_elapsed_time'] = (self.tasks[task_id]['time_returned'] -\n                                                  self.tasks[task_id]['time_submitted']).total_seconds()\n        if fail_mode is not None:\n            task_log_info['task_fail_mode'] = fail_mode\n        return task_log_info", "response": "Create the dictionary that will be included in the log."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef handle_app_update(self, task_id, future, memo_cbk=False):\n\n        if not self.tasks[task_id]['app_fu'].done():\n            logger.error(\"Internal consistency error: app_fu is not done for task {}\".format(task_id))\n        if not self.tasks[task_id]['app_fu'] == future:\n            logger.error(\"Internal consistency error: callback future is not the app_fu in task structure, for task {}\".format(task_id))\n\n        if not memo_cbk:\n            # Update the memoizer with the new result if this is not a\n            # result from a memo lookup and the task has reached a terminal state.\n            self.memoizer.update_memo(task_id, self.tasks[task_id], future)\n\n            if self.checkpoint_mode == 'task_exit':\n                self.checkpoint(tasks=[task_id])\n\n        # Submit _*_stage_out tasks for output data futures that correspond with remote files\n        if (self.tasks[task_id]['app_fu'] and\n            self.tasks[task_id]['app_fu'].done() and\n            self.tasks[task_id]['app_fu'].exception() is None and\n            self.tasks[task_id]['executor'] != 'data_manager' and\n            self.tasks[task_id]['func_name'] != '_ftp_stage_in' and\n            self.tasks[task_id]['func_name'] != '_http_stage_in'):\n            for dfu in self.tasks[task_id]['app_fu'].outputs:\n                f = dfu.file_obj\n                if isinstance(f, File) and f.is_remote():\n                    self.data_manager.stage_out(f, self.tasks[task_id]['executor'])\n\n        return", "response": "This function is called by the app when an app future is in its final state. It will trigger the app processing such as checkpointing the files and stageout tasks."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nhandles the actual submission of the task to the executor layer. If the app task has the executors attributes not set (default=='all') the task is launched on a randomly selected executor from the list of executors. This behavior could later be updated to support binding to executors based on user specified criteria. If the app task specifies a particular set of executors, it will be targeted at those specific executors. Args: task_id (uuid string) : A uuid string that uniquely identifies the task executable (callable) : A callable object args (list of positional args) kwargs (arbitrary keyword arguments) Returns: Future that tracks the execution of the submitted executable", "response": "def launch_task(self, task_id, executable, *args, **kwargs):\n        \"\"\"Handle the actual submission of the task to the executor layer.\n\n        If the app task has the executors attributes not set (default=='all')\n        the task is launched on a randomly selected executor from the\n        list of executors. This behavior could later be updated to support\n        binding to executors based on user specified criteria.\n\n        If the app task specifies a particular set of executors, it will be\n        targeted at those specific executors.\n\n        Args:\n            task_id (uuid string) : A uuid string that uniquely identifies the task\n            executable (callable) : A callable object\n            args (list of positional args)\n            kwargs (arbitrary keyword arguments)\n\n\n        Returns:\n            Future that tracks the execution of the submitted executable\n        \"\"\"\n        self.tasks[task_id]['time_submitted'] = datetime.datetime.now()\n\n        hit, memo_fu = self.memoizer.check_memo(task_id, self.tasks[task_id])\n        if hit:\n            logger.info(\"Reusing cached result for task {}\".format(task_id))\n            return memo_fu\n\n        executor_label = self.tasks[task_id][\"executor\"]\n        try:\n            executor = self.executors[executor_label]\n        except Exception:\n            logger.exception(\"Task {} requested invalid executor {}: config is\\n{}\".format(task_id, executor_label, self._config))\n\n        if self.monitoring is not None and self.monitoring.resource_monitoring_enabled:\n            executable = self.monitoring.monitor_wrapper(executable, task_id,\n                                                         self.monitoring.monitoring_hub_url,\n                                                         self.run_id,\n                                                         self.monitoring.resource_monitoring_interval)\n\n        with self.submitter_lock:\n            exec_fu = executor.submit(executable, *args, **kwargs)\n        self.tasks[task_id]['status'] = States.launched\n        if self.monitoring is not None:\n            task_log_info = self._create_task_log_info(task_id, 'lazy')\n            self.monitoring.send(MessageType.TASK_INFO, task_log_info)\n\n        exec_fu.retries_left = self._config.retries - \\\n            self.tasks[task_id]['fail_count']\n        logger.info(\"Task {} launched on executor {}\".format(task_id, executor.label))\n        return exec_fu"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nadd input dependencies to the inputs list.", "response": "def _add_input_deps(self, executor, args, kwargs):\n        \"\"\"Look for inputs of the app that are remote files. Submit stage_in\n        apps for such files and replace the file objects in the inputs list with\n        corresponding DataFuture objects.\n\n        Args:\n            - executor (str) : executor where the app is going to be launched\n            - args (List) : Positional args to app function\n            - kwargs (Dict) : Kwargs to app function\n        \"\"\"\n\n        # Return if the task is _*_stage_in\n        if executor == 'data_manager':\n            return args, kwargs\n\n        inputs = kwargs.get('inputs', [])\n        for idx, f in enumerate(inputs):\n            if isinstance(f, File) and f.is_remote():\n                inputs[idx] = self.data_manager.stage_in(f, executor)\n\n        for kwarg, f in kwargs.items():\n            if isinstance(f, File) and f.is_remote():\n                kwargs[kwarg] = self.data_manager.stage_in(f, executor)\n\n        newargs = list(args)\n        for idx, f in enumerate(newargs):\n            if isinstance(f, File) and f.is_remote():\n                newargs[idx] = self.data_manager.stage_in(f, executor)\n\n        return tuple(newargs), kwargs"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncount the number of unresolved futures on which a task depends.", "response": "def _gather_all_deps(self, args, kwargs):\n        \"\"\"Count the number of unresolved futures on which a task depends.\n\n        Args:\n            - args (List[args]) : The list of args list to the fn\n            - kwargs (Dict{kwargs}) : The dict of all kwargs passed to the fn\n\n        Returns:\n            - count, [list of dependencies]\n\n        \"\"\"\n        # Check the positional args\n        depends = []\n        count = 0\n        for dep in args:\n            if isinstance(dep, Future):\n                if self.tasks[dep.tid]['status'] not in FINAL_STATES:\n                    count += 1\n                depends.extend([dep])\n\n        # Check for explicit kwargs ex, fu_1=<fut>\n        for key in kwargs:\n            dep = kwargs[key]\n            if isinstance(dep, Future):\n                if self.tasks[dep.tid]['status'] not in FINAL_STATES:\n                    count += 1\n                depends.extend([dep])\n\n        # Check for futures in inputs=[<fut>...]\n        for dep in kwargs.get('inputs', []):\n            if isinstance(dep, Future):\n                if self.tasks[dep.tid]['status'] not in FINAL_STATES:\n                    count += 1\n                depends.extend([dep])\n\n        return count, depends"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef sanitize_and_wrap(self, task_id, args, kwargs):\n        dep_failures = []\n\n        # Replace item in args\n        new_args = []\n        for dep in args:\n            if isinstance(dep, Future):\n                try:\n                    new_args.extend([dep.result()])\n                except Exception as e:\n                    if self.tasks[dep.tid]['status'] in FINAL_FAILURE_STATES:\n                        dep_failures.extend([e])\n            else:\n                new_args.extend([dep])\n\n        # Check for explicit kwargs ex, fu_1=<fut>\n        for key in kwargs:\n            dep = kwargs[key]\n            if isinstance(dep, Future):\n                try:\n                    kwargs[key] = dep.result()\n                except Exception as e:\n                    if self.tasks[dep.tid]['status'] in FINAL_FAILURE_STATES:\n                        dep_failures.extend([e])\n\n        # Check for futures in inputs=[<fut>...]\n        if 'inputs' in kwargs:\n            new_inputs = []\n            for dep in kwargs['inputs']:\n                if isinstance(dep, Future):\n                    try:\n                        new_inputs.extend([dep.result()])\n                    except Exception as e:\n                        if self.tasks[dep.tid]['status'] in FINAL_FAILURE_STATES:\n                            dep_failures.extend([e])\n\n                else:\n                    new_inputs.extend([dep])\n            kwargs['inputs'] = new_inputs\n\n        return new_args, kwargs, dep_failures", "response": "This function should be called only when all the futures have been resolved."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef submit(self, func, *args, executors='all', fn_hash=None, cache=False, **kwargs):\n\n        if self.cleanup_called:\n            raise ValueError(\"Cannot submit to a DFK that has been cleaned up\")\n\n        task_id = self.task_count\n        self.task_count += 1\n        if isinstance(executors, str) and executors.lower() == 'all':\n            choices = list(e for e in self.executors if e != 'data_manager')\n        elif isinstance(executors, list):\n            choices = executors\n        executor = random.choice(choices)\n\n        # Transform remote input files to data futures\n        args, kwargs = self._add_input_deps(executor, args, kwargs)\n\n        task_def = {'depends': None,\n                    'executor': executor,\n                    'func': func,\n                    'func_name': func.__name__,\n                    'args': args,\n                    'kwargs': kwargs,\n                    'fn_hash': fn_hash,\n                    'memoize': cache,\n                    'callback': None,\n                    'exec_fu': None,\n                    'checkpoint': None,\n                    'fail_count': 0,\n                    'fail_history': [],\n                    'env': None,\n                    'status': States.unsched,\n                    'id': task_id,\n                    'time_submitted': None,\n                    'time_returned': None,\n                    'app_fu': None}\n\n        if task_id in self.tasks:\n            raise DuplicateTaskError(\n                \"internal consistency error: Task {0} already exists in task list\".format(task_id))\n        else:\n            self.tasks[task_id] = task_def\n\n        # Get the dep count and a list of dependencies for the task\n        dep_cnt, depends = self._gather_all_deps(args, kwargs)\n        self.tasks[task_id]['depends'] = depends\n\n        # Extract stdout and stderr to pass to AppFuture:\n        task_stdout = kwargs.get('stdout')\n        task_stderr = kwargs.get('stderr')\n\n        logger.info(\"Task {} submitted for App {}, waiting on tasks {}\".format(task_id,\n                                                                               task_def['func_name'],\n                                                                               [fu.tid for fu in depends]))\n\n        self.tasks[task_id]['task_launch_lock'] = threading.Lock()\n        app_fu = AppFuture(tid=task_id,\n                           stdout=task_stdout,\n                           stderr=task_stderr)\n\n        self.tasks[task_id]['app_fu'] = app_fu\n        app_fu.add_done_callback(partial(self.handle_app_update, task_id))\n        self.tasks[task_id]['status'] = States.pending\n        logger.debug(\"Task {} set to pending state with AppFuture: {}\".format(task_id, task_def['app_fu']))\n\n        # at this point add callbacks to all dependencies to do a launch_if_ready\n        # call whenever a dependency completes.\n\n        # we need to be careful about the order of setting the state to pending,\n        # adding the callbacks, and caling launch_if_ready explicitly once always below.\n\n        # I think as long as we call launch_if_ready once after setting pending, then\n        # we can add the callback dependencies at any point: if the callbacks all fire\n        # before then, they won't cause a launch, but the one below will. if they fire\n        # after we set it pending, then the last one will cause a launch, and the\n        # explicit one won't.\n\n        for d in depends:\n\n            def callback_adapter(dep_fut):\n                self.launch_if_ready(task_id)\n\n            try:\n                d.add_done_callback(callback_adapter)\n            except Exception as e:\n                logger.error(\"add_done_callback got an exception {} which will be ignored\".format(e))\n\n        self.launch_if_ready(task_id)\n\n        return task_def['app_fu']", "response": "Add a task to the dataflow system."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nwaiting for all tasks in the task list to be completed.", "response": "def wait_for_current_tasks(self):\n        \"\"\"Waits for all tasks in the task list to be completed, by waiting for their\n        AppFuture to be completed. This method will not necessarily wait for any tasks\n        added after cleanup has started (such as data stageout?)\n        \"\"\"\n\n        logger.info(\"Waiting for all remaining tasks to complete\")\n        for task_id in self.tasks:\n            # .exception() is a less exception throwing way of\n            # waiting for completion than .result()\n            fut = self.tasks[task_id]['app_fu']\n            if not fut.done():\n                logger.debug(\"Waiting for task {} to complete\".format(task_id))\n                fut.exception()\n        logger.info(\"All remaining tasks completed\")"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef cleanup(self):\n        logger.info(\"DFK cleanup initiated\")\n\n        # this check won't detect two DFK cleanups happening from\n        # different threads extremely close in time because of\n        # non-atomic read/modify of self.cleanup_called\n        if self.cleanup_called:\n            raise Exception(\"attempt to clean up DFK when it has already been cleaned-up\")\n        self.cleanup_called = True\n\n        self.log_task_states()\n\n        # Checkpointing takes priority over the rest of the tasks\n        # checkpoint if any valid checkpoint method is specified\n        if self.checkpoint_mode is not None:\n            self.checkpoint()\n\n            if self._checkpoint_timer:\n                logger.info(\"Stopping checkpoint timer\")\n                self._checkpoint_timer.close()\n\n        # Send final stats\n        self.usage_tracker.send_message()\n        self.usage_tracker.close()\n\n        logger.info(\"Terminating flow_control and strategy threads\")\n        self.flowcontrol.close()\n\n        for executor in self.executors.values():\n            if executor.managed:\n                if executor.scaling_enabled:\n                    job_ids = executor.provider.resources.keys()\n                    executor.scale_in(len(job_ids))\n                executor.shutdown()\n\n        self.time_completed = datetime.datetime.now()\n\n        if self.monitoring:\n            self.monitoring.send(MessageType.WORKFLOW_INFO,\n                                 {'tasks_failed_count': self.tasks_failed_count,\n                                  'tasks_completed_count': self.tasks_completed_count,\n                                  \"time_began\": self.time_began,\n                                  'time_completed': self.time_completed,\n                                  'workflow_duration': (self.time_completed - self.time_began).total_seconds(),\n                                  'run_id': self.run_id, 'rundir': self.run_dir})\n\n            self.monitoring.close()\n\n        \"\"\"\n        if self.logging_server is not None:\n            self.logging_server.terminate()\n            self.logging_server.join()\n\n        if self.web_app is not None:\n            self.web_app.terminate()\n            self.web_app.join()\n        \"\"\"\n        logger.info(\"DFK cleanup complete\")", "response": "This function cleans up the data flow kernel and all of its executors and the tasks in the pool."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nload a checkpoint file into a memoized lookup table.", "response": "def _load_checkpoints(self, checkpointDirs):\n        \"\"\"Load a checkpoint file into a lookup table.\n\n        The data being loaded from the pickle file mostly contains input\n        attributes of the task: func, args, kwargs, env...\n        To simplify the check of whether the exact task has been completed\n        in the checkpoint, we hash these input params and use it as the key\n        for the memoized lookup table.\n\n        Args:\n            - checkpointDirs (list) : List of filepaths to checkpoints\n              Eg. ['runinfo/001', 'runinfo/002']\n\n        Returns:\n            - memoized_lookup_table (dict)\n        \"\"\"\n        memo_lookup_table = {}\n\n        for checkpoint_dir in checkpointDirs:\n            logger.info(\"Loading checkpoints from {}\".format(checkpoint_dir))\n            checkpoint_file = os.path.join(checkpoint_dir, 'tasks.pkl')\n            try:\n                with open(checkpoint_file, 'rb') as f:\n                    while True:\n                        try:\n                            data = pickle.load(f)\n                            # Copy and hash only the input attributes\n                            memo_fu = Future()\n                            if data['exception']:\n                                memo_fu.set_exception(data['exception'])\n                            else:\n                                memo_fu.set_result(data['result'])\n                            memo_lookup_table[data['hash']] = memo_fu\n\n                        except EOFError:\n                            # Done with the checkpoint file\n                            break\n            except FileNotFoundError:\n                reason = \"Checkpoint file was not found: {}\".format(\n                    checkpoint_file)\n                logger.error(reason)\n                raise BadCheckpoint(reason)\n            except Exception:\n                reason = \"Failed to load checkpoint: {}\".format(\n                    checkpoint_file)\n                logger.error(reason)\n                raise BadCheckpoint(reason)\n\n            logger.info(\"Completed loading checkpoint:{0} with {1} tasks\".format(checkpoint_file,\n                                                                                 len(memo_lookup_table.keys())))\n        return memo_lookup_table"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nloading checkpoints from the checkpointDirs into a dictionary.", "response": "def load_checkpoints(self, checkpointDirs):\n        \"\"\"Load checkpoints from the checkpoint files into a dictionary.\n\n        The results are used to pre-populate the memoizer's lookup_table\n\n        Kwargs:\n             - checkpointDirs (list) : List of run folder to use as checkpoints\n               Eg. ['runinfo/001', 'runinfo/002']\n\n        Returns:\n             - dict containing, hashed -> future mappings\n        \"\"\"\n        self.memo_lookup_table = None\n\n        if not checkpointDirs:\n            return {}\n\n        if type(checkpointDirs) is not list:\n            raise BadCheckpoint(\"checkpointDirs expects a list of checkpoints\")\n\n        return self._load_checkpoints(checkpointDirs)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef load(cls, config: Optional[Config] = None):\n        if cls._dfk is not None:\n            raise RuntimeError('Config has already been loaded')\n\n        if config is None:\n            cls._dfk = DataFlowKernel(Config())\n        else:\n            cls._dfk = DataFlowKernel(config)\n\n        return cls._dfk", "response": "Loads a DataFlowKernel object."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nstarts the interchange process", "response": "def starter(comm_q, *args, **kwargs):\n    \"\"\"Start the interchange process\n\n    The executor is expected to call this function. The args, kwargs match that of the Interchange.__init__\n    \"\"\"\n    # logger = multiprocessing.get_logger()\n    ic = Interchange(*args, **kwargs)\n    comm_q.put((ic.worker_task_port,\n                ic.worker_result_port))\n    ic.start()"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nget a batch of tasks from the internal pending_task_queue and returns them as a list of dicts.", "response": "def get_tasks(self, count):\n        \"\"\" Obtains a batch of tasks from the internal pending_task_queue\n\n        Parameters\n        ----------\n        count: int\n            Count of tasks to get from the queue\n\n        Returns\n        -------\n        List of upto count tasks. May return fewer than count down to an empty list\n            eg. [{'task_id':<x>, 'buffer':<buf>} ... ]\n        \"\"\"\n        tasks = []\n        for i in range(0, count):\n            try:\n                x = self.pending_task_queue.get(block=False)\n            except queue.Empty:\n                break\n            else:\n                tasks.append(x)\n\n        return tasks"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\npulling tasks from the incoming tasks 0mq pipe onto the internal task queue and put them into the pending task queue.", "response": "def migrate_tasks_to_internal(self, kill_event):\n        \"\"\"Pull tasks from the incoming tasks 0mq pipe onto the internal\n        pending task queue\n\n        Parameters:\n        -----------\n        kill_event : threading.Event\n              Event to let the thread know when it is time to die.\n        \"\"\"\n        logger.info(\"[TASK_PULL_THREAD] Starting\")\n        task_counter = 0\n        poller = zmq.Poller()\n        poller.register(self.task_incoming, zmq.POLLIN)\n\n        while not kill_event.is_set():\n            try:\n                msg = self.task_incoming.recv_pyobj()\n            except zmq.Again:\n                # We just timed out while attempting to receive\n                logger.debug(\"[TASK_PULL_THREAD] {} tasks in internal queue\".format(self.pending_task_queue.qsize()))\n                continue\n\n            if msg == 'STOP':\n                kill_event.set()\n                break\n            else:\n                self.pending_task_queue.put(msg)\n                task_counter += 1\n                logger.debug(\"[TASK_PULL_THREAD] Fetched task:{}\".format(task_counter))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef _command_server(self, kill_event):\n        logger.debug(\"[COMMAND] Command Server Starting\")\n        while not kill_event.is_set():\n            try:\n                command_req = self.command_channel.recv_pyobj()\n                logger.debug(\"[COMMAND] Received command request: {}\".format(command_req))\n                if command_req == \"OUTSTANDING_C\":\n                    outstanding = self.pending_task_queue.qsize()\n                    for manager in self._ready_manager_queue:\n                        outstanding += len(self._ready_manager_queue[manager]['tasks'])\n                    reply = outstanding\n\n                elif command_req == \"WORKERS\":\n                    num_workers = 0\n                    for manager in self._ready_manager_queue:\n                        num_workers += self._ready_manager_queue[manager]['worker_count']\n                    reply = num_workers\n                elif command_req == \"MANAGERS\":\n                    reply = []\n                    for manager in self._ready_manager_queue:\n                        resp = {'manager': manager.decode('utf-8'),\n                                'block_id': self._ready_manager_queue[manager]['block_id'],\n                                'worker_count': self._ready_manager_queue[manager]['worker_count'],\n                                'tasks': len(self._ready_manager_queue[manager]['tasks']),\n                                'active': self._ready_manager_queue[manager]['active']}\n                        reply.append(resp)\n\n                elif command_req.startswith(\"HOLD_WORKER\"):\n                    cmd, s_manager = command_req.split(';')\n                    manager = s_manager.encode('utf-8')\n                    logger.info(\"[CMD] Received HOLD_WORKER for {}\".format(manager))\n                    if manager in self._ready_manager_queue:\n                        self._ready_manager_queue[manager]['active'] = False\n                        reply = True\n                    else:\n                        reply = False\n\n                elif command_req == \"SHUTDOWN\":\n                    logger.info(\"[CMD] Received SHUTDOWN command\")\n                    kill_event.set()\n                    reply = True\n\n                else:\n                    reply = None\n\n                logger.debug(\"[COMMAND] Reply: {}\".format(reply))\n                self.command_channel.send_pyobj(reply)\n\n            except zmq.Again:\n                logger.debug(\"[COMMAND] is alive\")\n                continue", "response": "Command server to run async command"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef start(self, poll_period=None):\n        logger.info(\"Incoming ports bound\")\n\n        if poll_period is None:\n            poll_period = self.poll_period\n\n        start = time.time()\n        count = 0\n\n        self._kill_event = threading.Event()\n        self._task_puller_thread = threading.Thread(target=self.migrate_tasks_to_internal,\n                                                    args=(self._kill_event,))\n        self._task_puller_thread.start()\n\n        self._command_thread = threading.Thread(target=self._command_server,\n                                                args=(self._kill_event,))\n        self._command_thread.start()\n\n        poller = zmq.Poller()\n        # poller.register(self.task_incoming, zmq.POLLIN)\n        poller.register(self.task_outgoing, zmq.POLLIN)\n        poller.register(self.results_incoming, zmq.POLLIN)\n\n        # These are managers which we should examine in an iteration\n        # for scheduling a job (or maybe any other attention?).\n        # Anything altering the state of the manager should add it\n        # onto this list.\n        interesting_managers = set()\n\n        while not self._kill_event.is_set():\n            self.socks = dict(poller.poll(timeout=poll_period))\n\n            # Listen for requests for work\n            if self.task_outgoing in self.socks and self.socks[self.task_outgoing] == zmq.POLLIN:\n                logger.debug(\"[MAIN] starting task_outgoing section\")\n                message = self.task_outgoing.recv_multipart()\n                manager = message[0]\n\n                if manager not in self._ready_manager_queue:\n                    reg_flag = False\n\n                    try:\n                        msg = json.loads(message[1].decode('utf-8'))\n                        reg_flag = True\n                    except Exception:\n                        logger.warning(\"[MAIN] Got a non-json registration message from manager:{}\".format(\n                            manager))\n                        logger.debug(\"[MAIN] Message :\\n{}\\n\".format(message[0]))\n\n                    # By default we set up to ignore bad nodes/registration messages.\n                    self._ready_manager_queue[manager] = {'last': time.time(),\n                                                          'free_capacity': 0,\n                                                          'block_id': None,\n                                                          'max_capacity': 0,\n                                                          'active': True,\n                                                          'tasks': []}\n                    if reg_flag is True:\n                        interesting_managers.add(manager)\n                        logger.info(\"[MAIN] Adding manager: {} to ready queue\".format(manager))\n                        self._ready_manager_queue[manager].update(msg)\n                        logger.info(\"[MAIN] Registration info for manager {}: {}\".format(manager, msg))\n\n                        if (msg['python_v'].rsplit(\".\", 1)[0] != self.current_platform['python_v'].rsplit(\".\", 1)[0] or\n                            msg['parsl_v'] != self.current_platform['parsl_v']):\n                            logger.warn(\"[MAIN] Manager {} has incompatible version info with the interchange\".format(manager))\n\n                            if self.suppress_failure is False:\n                                logger.debug(\"Setting kill event\")\n                                self._kill_event.set()\n                                e = ManagerLost(manager)\n                                result_package = {'task_id': -1, 'exception': serialize_object(e)}\n                                pkl_package = pickle.dumps(result_package)\n                                self.results_outgoing.send(pkl_package)\n                                logger.warning(\"[MAIN] Sent failure reports, unregistering manager\")\n                            else:\n                                logger.debug(\"[MAIN] Suppressing shutdown due to version incompatibility\")\n                        else:\n                            logger.info(\"[MAIN] Manager {} has compatible Parsl version {}\".format(manager, msg['parsl_v']))\n                            logger.info(\"[MAIN] Manager {} has compatible Python version {}\".format(manager,\n                                                                                                    msg['python_v'].rsplit(\".\", 1)[0]))\n                    else:\n                        # Registration has failed.\n                        if self.suppress_failure is False:\n                            self._kill_event.set()\n                            e = BadRegistration(manager, critical=True)\n                            result_package = {'task_id': -1, 'exception': serialize_object(e)}\n                            pkl_package = pickle.dumps(result_package)\n                            self.results_outgoing.send(pkl_package)\n                        else:\n                            logger.debug(\"[MAIN] Suppressing bad registration from manager:{}\".format(\n                                manager))\n\n                else:\n                    tasks_requested = int.from_bytes(message[1], \"little\")\n                    self._ready_manager_queue[manager]['last'] = time.time()\n                    if tasks_requested == HEARTBEAT_CODE:\n                        logger.debug(\"[MAIN] Manager {} sent heartbeat\".format(manager))\n                        self.task_outgoing.send_multipart([manager, b'', PKL_HEARTBEAT_CODE])\n                    else:\n                        logger.debug(\"[MAIN] Manager {} requested {} tasks\".format(manager, tasks_requested))\n                        self._ready_manager_queue[manager]['free_capacity'] = tasks_requested\n                        interesting_managers.add(manager)\n                logger.debug(\"[MAIN] leaving task_outgoing section\")\n\n            # If we had received any requests, check if there are tasks that could be passed\n\n            logger.debug(\"Managers count (total/interesting): {}/{}\".format(len(self._ready_manager_queue),\n                                                                            len(interesting_managers)))\n\n            if interesting_managers and not self.pending_task_queue.empty():\n                shuffled_managers = list(interesting_managers)\n                random.shuffle(shuffled_managers)\n\n                while shuffled_managers and not self.pending_task_queue.empty():  # cf. the if statement above...\n                    manager = shuffled_managers.pop()\n                    tasks_inflight = len(self._ready_manager_queue[manager]['tasks'])\n                    real_capacity = min(self._ready_manager_queue[manager]['free_capacity'],\n                                        self._ready_manager_queue[manager]['max_capacity'] - tasks_inflight)\n\n                    if (real_capacity and self._ready_manager_queue[manager]['active']):\n                        tasks = self.get_tasks(real_capacity)\n                        if tasks:\n                            self.task_outgoing.send_multipart([manager, b'', pickle.dumps(tasks)])\n                            task_count = len(tasks)\n                            count += task_count\n                            tids = [t['task_id'] for t in tasks]\n                            self._ready_manager_queue[manager]['free_capacity'] -= task_count\n                            self._ready_manager_queue[manager]['tasks'].extend(tids)\n                            logger.debug(\"[MAIN] Sent tasks: {} to manager {}\".format(tids, manager))\n                            if self._ready_manager_queue[manager]['free_capacity'] > 0:\n                                logger.debug(\"[MAIN] Manager {} has free_capacity {}\".format(manager, self._ready_manager_queue[manager]['free_capacity']))\n                                # ... so keep it in the interesting_managers list\n                            else:\n                                logger.debug(\"[MAIN] Manager {} is now saturated\".format(manager))\n                                interesting_managers.remove(manager)\n                    else:\n                        interesting_managers.remove(manager)\n                        # logger.debug(\"Nothing to send to manager {}\".format(manager))\n                logger.debug(\"[MAIN] leaving _ready_manager_queue section, with {} managers still interesting\".format(len(interesting_managers)))\n            else:\n                logger.debug(\"[MAIN] either no interesting managers or no tasks, so skipping manager pass\")\n            # Receive any results and forward to client\n            if self.results_incoming in self.socks and self.socks[self.results_incoming] == zmq.POLLIN:\n                logger.debug(\"[MAIN] entering results_incoming section\")\n                manager, *b_messages = self.results_incoming.recv_multipart()\n                if manager not in self._ready_manager_queue:\n                    logger.warning(\"[MAIN] Received a result from a un-registered manager: {}\".format(manager))\n                else:\n                    logger.debug(\"[MAIN] Got {} result items in batch\".format(len(b_messages)))\n                    for b_message in b_messages:\n                        r = pickle.loads(b_message)\n                        # logger.debug(\"[MAIN] Received result for task {} from {}\".format(r['task_id'], manager))\n                        self._ready_manager_queue[manager]['tasks'].remove(r['task_id'])\n                    self.results_outgoing.send_multipart(b_messages)\n                    logger.debug(\"[MAIN] Current tasks: {}\".format(self._ready_manager_queue[manager]['tasks']))\n                logger.debug(\"[MAIN] leaving results_incoming section\")\n\n            logger.debug(\"[MAIN] entering bad_managers section\")\n            bad_managers = [manager for manager in self._ready_manager_queue if\n                            time.time() - self._ready_manager_queue[manager]['last'] > self.heartbeat_threshold]\n            for manager in bad_managers:\n                logger.debug(\"[MAIN] Last: {} Current: {}\".format(self._ready_manager_queue[manager]['last'], time.time()))\n                logger.warning(\"[MAIN] Too many heartbeats missed for manager {}\".format(manager))\n\n                for tid in self._ready_manager_queue[manager]['tasks']:\n                    try:\n                        raise ManagerLost(manager)\n                    except Exception:\n                        result_package = {'task_id': tid, 'exception': serialize_object(RemoteExceptionWrapper(*sys.exc_info()))}\n                        pkl_package = pickle.dumps(result_package)\n                        self.results_outgoing.send(pkl_package)\n                        logger.warning(\"[MAIN] Sent failure reports, unregistering manager\")\n                self._ready_manager_queue.pop(manager, 'None')\n            logger.debug(\"[MAIN] leaving bad_managers section\")\n            logger.debug(\"[MAIN] ending one main loop iteration\")\n\n        delta = time.time() - start\n        logger.info(\"Processed {} tasks in {} seconds\".format(count, delta))\n        logger.warning(\"Exiting\")", "response": "Start the NeedNameQeueu thread."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef starter(comm_q, *args, **kwargs):\n    # logger = multiprocessing.get_logger()\n    ic = Interchange(*args, **kwargs)\n    comm_q.put(ic.worker_port)\n    ic.start()\n    logger.debug(\"Port information sent back to client\")", "response": "Start the interchange process"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef start(self):\n        logger.info(\"Starting interchange\")\n        # last = time.time()\n\n        while True:\n            # active_flag = False\n            socks = dict(self.poller.poll(1))\n\n            if socks.get(self.task_incoming) == zmq.POLLIN:\n                message = self.task_incoming.recv_multipart()\n                logger.debug(\"Got new task from client\")\n                self.worker_messages.send_multipart(message)\n                logger.debug(\"Sent task to worker\")\n                # active_flag = True\n                # last = time.time()\n\n            if socks.get(self.worker_messages) == zmq.POLLIN:\n                message = self.worker_messages.recv_multipart()\n                logger.debug(\"Got new result from worker\")\n                # self.result_outgoing.send_multipart(message)\n                self.result_outgoing.send_multipart(message[1:])\n\n                logger.debug(\"Sent result to client\")", "response": "Start the interchange loop."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef execute_task(bufs):\n    user_ns = locals()\n    user_ns.update({'__builtins__': __builtins__})\n\n    f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)\n\n    # We might need to look into callability of the function from itself\n    # since we change it's name in the new namespace\n    prefix = \"parsl_\"\n    fname = prefix + \"f\"\n    argname = prefix + \"args\"\n    kwargname = prefix + \"kwargs\"\n    resultname = prefix + \"result\"\n\n    user_ns.update({fname: f,\n                    argname: args,\n                    kwargname: kwargs,\n                    resultname: resultname})\n\n    code = \"{0} = {1}(*{2}, **{3})\".format(resultname, fname,\n                                           argname, kwargname)\n    try:\n        # logger.debug(\"[RUNNER] Executing: {0}\".format(code))\n        exec(code, user_ns, user_ns)\n\n    except Exception as e:\n        logger.warning(\"Caught exception; will raise it: {}\".format(e), exc_info=True)\n        raise e\n\n    else:\n        # logger.debug(\"[RUNNER] Result: {0}\".format(user_ns.get(resultname)))\n        return user_ns.get(resultname)", "response": "Deserialize the buffer and execute the task."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nlisten on the pending_result_queue and sends out results via 0mq", "response": "def push_results(self, kill_event):\n        \"\"\" Listens on the pending_result_queue and sends out results via 0mq\n\n        Parameters:\n        -----------\n        kill_event : threading.Event\n              Event to let the thread know when it is time to die.\n        \"\"\"\n\n        logger.debug(\"[RESULT_PUSH_THREAD] Starting thread\")\n\n        push_poll_period = max(10, self.poll_period) / 1000    # push_poll_period must be atleast 10 ms\n        logger.debug(\"[RESULT_PUSH_THREAD] push poll period: {}\".format(push_poll_period))\n\n        last_beat = time.time()\n        items = []\n\n        while not kill_event.is_set():\n\n            try:\n                r = self.pending_result_queue.get(block=True, timeout=push_poll_period)\n                items.append(r)\n            except queue.Empty:\n                pass\n            except Exception as e:\n                logger.exception(\"[RESULT_PUSH_THREAD] Got an exception: {}\".format(e))\n\n            # If we have reached poll_period duration or timer has expired, we send results\n            if len(items) >= self.max_queue_size or time.time() > last_beat + push_poll_period:\n                last_beat = time.time()\n                if items:\n                    self.result_outgoing.send_multipart(items)\n                    items = []\n\n        logger.critical(\"[RESULT_PUSH_THREAD] Exiting\")"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nstart the worker processes.", "response": "def start(self):\n        \"\"\" Start the worker processes.\n\n        TODO: Move task receiving to a thread\n        \"\"\"\n        start = time.time()\n        self._kill_event = threading.Event()\n\n        self.procs = {}\n        for worker_id in range(self.worker_count):\n            p = multiprocessing.Process(target=worker, args=(worker_id,\n                                                             self.uid,\n                                                             self.pending_task_queue,\n                                                             self.pending_result_queue,\n                                                             self.ready_worker_queue,\n                                                         ))\n            p.start()\n            self.procs[worker_id] = p\n\n        logger.debug(\"Manager synced with workers\")\n\n        self._task_puller_thread = threading.Thread(target=self.pull_tasks,\n                                                    args=(self._kill_event,))\n        self._result_pusher_thread = threading.Thread(target=self.push_results,\n                                                      args=(self._kill_event,))\n        self._task_puller_thread.start()\n        self._result_pusher_thread.start()\n\n        logger.info(\"Loop start\")\n\n        # TODO : Add mechanism in this loop to stop the worker pool\n        # This might need a multiprocessing event to signal back.\n        self._kill_event.wait()\n        logger.critical(\"[MAIN] Received kill event, terminating worker processes\")\n\n        self._task_puller_thread.join()\n        self._result_pusher_thread.join()\n        for proc_id in self.procs:\n            self.procs[proc_id].terminate()\n            logger.critical(\"Terminating worker {}:{}\".format(self.procs[proc_id],\n                                                              self.procs[proc_id].is_alive()))\n            self.procs[proc_id].join()\n            logger.debug(\"Worker:{} joined successfully\".format(self.procs[proc_id]))\n\n        self.task_incoming.close()\n        self.result_outgoing.close()\n        self.context.term()\n        delta = time.time() - start\n        logger.info(\"process_worker_pool ran for {} seconds\".format(delta))\n        return"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nreturn the DataManager of the currently loaded DataFlowKernel.", "response": "def get_data_manager(cls):\n        \"\"\"Return the DataManager of the currently loaded DataFlowKernel.\n        \"\"\"\n        from parsl.dataflow.dflow import DataFlowKernelLoader\n        dfk = DataFlowKernelLoader.dfk()\n\n        return dfk.executors['data_manager']"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nshuts down the executor", "response": "def shutdown(self, block=False):\n        \"\"\"Shutdown the ThreadPool.\n\n        Kwargs:\n            - block (bool): To block for confirmations or not\n\n        \"\"\"\n        x = self.executor.shutdown(wait=block)\n        logger.debug(\"Done with executor shutdown\")\n        return x"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ntransport the file from the input source to the executor. This function returns a DataFuture. Args: - self - file (File) : file to stage in - executor (str) : an executor the file is going to be staged in to. If the executor argument is not specified for a file with 'globus' scheme, the file will be staged in to the first executor with the \"globus\" key in a config.", "response": "def stage_in(self, file, executor):\n        \"\"\"Transport the file from the input source to the executor.\n\n        This function returns a DataFuture.\n\n        Args:\n            - self\n            - file (File) : file to stage in\n            - executor (str) : an executor the file is going to be staged in to.\n                                If the executor argument is not specified for a file\n                                with 'globus' scheme, the file will be staged in to\n                                the first executor with the \"globus\" key in a config.\n        \"\"\"\n\n        if file.scheme == 'ftp':\n            working_dir = self.dfk.executors[executor].working_dir\n            stage_in_app = self._ftp_stage_in_app(executor=executor)\n            app_fut = stage_in_app(working_dir, outputs=[file])\n            return app_fut._outputs[0]\n        elif file.scheme == 'http' or file.scheme == 'https':\n            working_dir = self.dfk.executors[executor].working_dir\n            stage_in_app = self._http_stage_in_app(executor=executor)\n            app_fut = stage_in_app(working_dir, outputs=[file])\n            return app_fut._outputs[0]\n        elif file.scheme == 'globus':\n            globus_ep = self._get_globus_endpoint(executor)\n            stage_in_app = self._globus_stage_in_app()\n            app_fut = stage_in_app(globus_ep, outputs=[file])\n            return app_fut._outputs[0]\n        else:\n            raise Exception('Staging in with unknown file scheme {} is not supported'.format(file.scheme))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef stage_out(self, file, executor):\n\n        if file.scheme == 'http' or file.scheme == 'https':\n            raise Exception('HTTP/HTTPS file staging out is not supported')\n        elif file.scheme == 'ftp':\n            raise Exception('FTP file staging out is not supported')\n        elif file.scheme == 'globus':\n            globus_ep = self._get_globus_endpoint(executor)\n            stage_out_app = self._globus_stage_out_app()\n            return stage_out_app(globus_ep, inputs=[file])\n        else:\n            raise Exception('Staging out with unknown file scheme {} is not supported'.format(file.scheme))", "response": "This function is used to transport the file from the local filesystem to the remote globus endpoint. This function returns a DataFuture that is returned when the file is staged out from the local filesystem."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreturning a list of checkpoints from all last runs in the specified directory.", "response": "def get_all_checkpoints(rundir=\"runinfo\"):\n    \"\"\"Finds the checkpoints from all last runs.\n\n    Note that checkpoints are incremental, and this helper will not find\n    previous checkpoints from earlier than the most recent run. It probably\n    should be made to do so.\n\n    Kwargs:\n       - rundir(str) : Path to the runinfo directory\n\n    Returns:\n       - a list suitable for the checkpointFiles parameter of DataFlowKernel\n         constructor\n\n    \"\"\"\n\n    if(not os.path.isdir(rundir)):\n        return []\n\n    dirs = sorted(os.listdir(rundir))\n\n    checkpoints = []\n\n    for runid in dirs:\n\n        checkpoint = os.path.abspath('{}/{}/checkpoint'.format(rundir, runid))\n\n        if os.path.isdir(checkpoint):\n            checkpoints.append(checkpoint)\n\n    return checkpoints"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_last_checkpoint(rundir=\"runinfo\"):\n    if not os.path.isdir(rundir):\n        return []\n\n    dirs = sorted(os.listdir(rundir))\n\n    if len(dirs) == 0:\n        return []\n\n    last_runid = dirs[-1]\n    last_checkpoint = os.path.abspath('{}/{}/checkpoint'.format(rundir, last_runid))\n\n    if(not(os.path.isdir(last_checkpoint))):\n        return []\n\n    return [last_checkpoint]", "response": "This helper function returns the last checkpoint from the last run of the data flow kernel."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef wtime_to_minutes(time_string):\n    ''' wtime_to_minutes\n\n    Convert standard wallclock time string to minutes.\n\n    Args:\n        - Time_string in HH:MM:SS format\n\n    Returns:\n        (int) minutes\n\n    '''\n    hours, mins, seconds = time_string.split(':')\n    total_mins = int(hours) * 60 + int(mins)\n    if total_mins < 1:\n        logger.warning(\"Time string '{}' parsed to {} minutes, less than 1\".format(time_string, total_mins))\n    return total_mins", "response": "Convert standard wallclock time string to minutes."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef interactive(f):\n    # build new FunctionType, so it can have the right globals\n    # interactive functions never have closures, that's kind of the point\n    if isinstance(f, FunctionType):\n        mainmod = __import__('__main__')\n        f = FunctionType(f.__code__, mainmod.__dict__,\n                         f.__name__, f.__defaults__,\n                         )\n    # associate with __main__ for uncanning\n    f.__module__ = '__main__'\n    return f", "response": "Decorator for making functions appear as interactively defined."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef use_pickle():\n    from . import serialize\n    serialize.pickle = serialize._stdlib_pickle\n\n    # restore special function handling\n    can_map[FunctionType] = _original_can_map[FunctionType]", "response": "Reverts custom serialization enabled by use_dill | cloudpickle.\n   "}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\nimports any string - keys in a type mapping.", "response": "def _import_mapping(mapping, original=None):\n    \"\"\"Import any string-keys in a type mapping.\"\"\"\n    #log = get_logger()\n    #log.debug(\"Importing canning map\")\n    for key, value in list(mapping.items()):\n        if isinstance(key, string_types):\n            try:\n                cls = import_item(key)\n            except Exception:\n                if original and key not in original:\n                    # only message on user-added classes\n                    # log.error(\"canning class not importable: %r\", key, exc_info=True)\n                    print(\"ERROR: canning class not importable: %r\", key, exc_info=True)\n                mapping.pop(key)\n            else:\n                mapping[cls] = mapping.pop(key)"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef istype(obj, check):\n    if isinstance(check, tuple):\n        for cls in check:\n            if type(obj) is cls:\n                return True\n        return False\n    else:\n        return type(obj) is check", "response": "Like isinstance but won t catch subclasses.\ninelistype"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\npreparing an object for pickling.", "response": "def can(obj):\n    \"\"\"Prepare an object for pickling.\"\"\"\n    import_needed = False\n\n    for cls, canner in iteritems(can_map):\n        if isinstance(cls, string_types):\n            import_needed = True\n            break\n        elif istype(obj, cls):\n            return canner(obj)\n\n    if import_needed:\n        # perform can_map imports, then try again\n        # this will usually only happen once\n        _import_mapping(can_map, _original_can_map)\n        return can(obj)\n\n    return obj"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nmutes newly added handlers to the root level right after calling executor. status", "response": "def unset_logging(self):\n        \"\"\" Mute newly added handlers to the root level, right after calling executor.status\n        \"\"\"\n        if self.logger_flag is True:\n            return\n\n        root_logger = logging.getLogger()\n\n        for hndlr in root_logger.handlers:\n            if hndlr not in self.prior_loghandlers:\n                hndlr.setLevel(logging.ERROR)\n\n        self.logger_flag = True"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef _strategy_simple(self, tasks, *args, kind=None, **kwargs):\n\n        for label, executor in self.dfk.executors.items():\n            if not executor.scaling_enabled:\n                continue\n\n            # Tasks that are either pending completion\n            active_tasks = executor.outstanding\n\n            status = executor.status()\n            self.unset_logging()\n\n            # FIXME we need to handle case where provider does not define these\n            # FIXME probably more of this logic should be moved to the provider\n            min_blocks = executor.provider.min_blocks\n            max_blocks = executor.provider.max_blocks\n            if isinstance(executor, IPyParallelExecutor):\n                tasks_per_node = executor.workers_per_node\n            elif isinstance(executor, HighThroughputExecutor):\n                # This is probably wrong calculation, we need this to come from the executor\n                # since we can't know slots ahead of time.\n                tasks_per_node = 1\n            elif isinstance(executor, ExtremeScaleExecutor):\n                tasks_per_node = executor.ranks_per_node\n\n            nodes_per_block = executor.provider.nodes_per_block\n            parallelism = executor.provider.parallelism\n\n            running = sum([1 for x in status if x == 'RUNNING'])\n            submitting = sum([1 for x in status if x == 'SUBMITTING'])\n            pending = sum([1 for x in status if x == 'PENDING'])\n            active_blocks = running + submitting + pending\n            active_slots = active_blocks * tasks_per_node * nodes_per_block\n\n            if hasattr(executor, 'connected_workers'):\n                logger.debug('Executor {} has {} active tasks, {}/{}/{} running/submitted/pending blocks, and {} connected workers'.format(\n                    label, active_tasks, running, submitting, pending, executor.connected_workers))\n            else:\n                logger.debug('Executor {} has {} active tasks and {}/{}/{} running/submitted/pending blocks'.format(\n                    label, active_tasks, running, submitting, pending))\n\n            # reset kill timer if executor has active tasks\n            if active_tasks > 0 and self.executors[executor.label]['idle_since']:\n                self.executors[executor.label]['idle_since'] = None\n\n            # Case 1\n            # No tasks.\n            if active_tasks == 0:\n                # Case 1a\n                # Fewer blocks that min_blocks\n                if active_blocks <= min_blocks:\n                    # Ignore\n                    # logger.debug(\"Strategy: Case.1a\")\n                    pass\n\n                # Case 1b\n                # More blocks than min_blocks. Scale down\n                else:\n                    # We want to make sure that max_idletime is reached\n                    # before killing off resources\n                    if not self.executors[executor.label]['idle_since']:\n                        logger.debug(\"Executor {} has 0 active tasks; starting kill timer (if idle time exceeds {}s, resources will be removed)\".format(\n                            label, self.max_idletime)\n                        )\n                        self.executors[executor.label]['idle_since'] = time.time()\n\n                    idle_since = self.executors[executor.label]['idle_since']\n                    if (time.time() - idle_since) > self.max_idletime:\n                        # We have resources idle for the max duration,\n                        # we have to scale_in now.\n                        logger.debug(\"Idle time has reached {}s for executor {}; removing resources\".format(\n                            self.max_idletime, label)\n                        )\n                        executor.scale_in(active_blocks - min_blocks)\n\n                    else:\n                        pass\n                        # logger.debug(\"Strategy: Case.1b. Waiting for timer : {0}\".format(idle_since))\n\n            # Case 2\n            # More tasks than the available slots.\n            elif (float(active_slots) / active_tasks) < parallelism:\n                # Case 2a\n                # We have the max blocks possible\n                if active_blocks >= max_blocks:\n                    # Ignore since we already have the max nodes\n                    # logger.debug(\"Strategy: Case.2a\")\n                    pass\n\n                # Case 2b\n                else:\n                    # logger.debug(\"Strategy: Case.2b\")\n                    excess = math.ceil((active_tasks * parallelism) - active_slots)\n                    excess_blocks = math.ceil(float(excess) / (tasks_per_node * nodes_per_block))\n                    logger.debug(\"Requesting {} more blocks\".format(excess_blocks))\n                    executor.scale_out(excess_blocks)\n\n            elif active_slots == 0 and active_tasks > 0:\n                # Case 4\n                # Check if slots are being lost quickly ?\n                logger.debug(\"Requesting single slot\")\n                executor.scale_out(1)\n            # Case 3\n            # tasks ~ slots\n            else:\n                # logger.debug(\"Strategy: Case 3\")\n                pass", "response": "Simple strategy for the DFK and executors."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef transfer_file(cls, src_ep, dst_ep, src_path, dst_path):\n        tc = globus_sdk.TransferClient(authorizer=cls.authorizer)\n        td = globus_sdk.TransferData(tc, src_ep, dst_ep)\n        td.add_item(src_path, dst_path)\n        try:\n            task = tc.submit_transfer(td)\n        except Exception as e:\n            raise Exception('Globus transfer from {}{} to {}{} failed due to error: {}'.format(\n                src_ep, src_path, dst_ep, dst_path, e))\n\n        last_event_time = None\n        \"\"\"\n        A Globus transfer job (task) can be in one of the three states: ACTIVE, SUCCEEDED, FAILED.\n        Parsl every 20 seconds polls a status of the transfer job (task) from the Globus Transfer service,\n        with 60 second timeout limit. If the task is ACTIVE after time runs out 'task_wait' returns False,\n        and True otherwise.\n        \"\"\"\n        while not tc.task_wait(task['task_id'], 60, 15):\n            task = tc.get_task(task['task_id'])\n            # Get the last error Globus event\n            events = tc.task_event_list(task['task_id'], num_results=1, filter='is_error:1')\n            event = events.data[0]\n            # Print the error event to stderr and Parsl file log if it was not yet printed\n            if event['time'] != last_event_time:\n                last_event_time = event['time']\n                logger.warn('Non-critical Globus Transfer error event for globus://{}{}: \"{}\" at {}. Retrying...'.format(\n                    src_ep, src_path, event['description'], event['time']))\n                logger.debug('Globus Transfer error details: {}'.format(event['details']))\n\n        \"\"\"\n        The Globus transfer job (task) has been terminated (is not ACTIVE). Check if the transfer\n        SUCCEEDED or FAILED.\n        \"\"\"\n        task = tc.get_task(task['task_id'])\n        if task['status'] == 'SUCCEEDED':\n            logger.debug('Globus transfer {}, from {}{} to {}{} succeeded'.format(\n                task['task_id'], src_ep, src_path, dst_ep, dst_path))\n        else:\n            logger.debug('Globus Transfer task: {}'.format(task))\n            events = tc.task_event_list(task['task_id'], num_results=1, filter='is_error:1')\n            event = events.data[0]\n            raise Exception('Globus transfer {}, from {}{} to {}{} failed due to error: \"{}\"'.format(\n                task['task_id'], src_ep, src_path, dst_ep, dst_path, event['details']))", "response": "This method is used to transfer a file from one endpoint to another."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_parsl_logger(\n                  logger_name='parsl_monitor_logger',\n                  is_logging_server=False,\n                  monitoring_config=None,\n                  **kwargs):\n    \"\"\"\n    Parameters\n    ----------\n    logger_name : str, optional\n        Name of the logger to use. Prevents adding repeat handlers or incorrect handlers\n    is_logging_server : Bool, optional\n        Used internally to determine which handler to return when using local db logging\n    monitoring_config : MonitoringConfig, optional\n        Pass in a logger class object to use for generating loggers.\n\n    Returns\n    -------\n    logging.logger object\n\n    Raises\n    ------\n    OptionalModuleMissing\n\n    \"\"\"\n\n    logger = logging.getLogger(logger_name)\n\n    if monitoring_config is None:\n        logger.addHandler(NullHandler())\n        return logger\n\n    if monitoring_config.store is None:\n        raise ValueError('No MonitoringStore defined')\n\n    if is_logging_server:\n        # add a handler that will take logs being received on the server and log them to the store\n        handler = DatabaseHandler(monitoring_config.store.connection_string)\n        # use the specific name generated by the server or the monitor wrapper\n        logger = logging.getLogger(logger_name)\n        logger.setLevel(logging.INFO)\n        logger.addHandler(handler)\n    else:\n        # add a handler that will pass logs to the logging server\n        handler = RemoteHandler(monitoring_config.store.logging_server_host, monitoring_config.store.logging_server_port)\n        # use the specific name generated by the server or the monitor wrapper\n        logger = logging.getLogger(logger_name)\n        logger.setLevel(logging.INFO)\n        logger.addHandler(handler)\n\n    return logger", "response": "Returns a new parsl logger object."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef start(self):\n\n        if self.mode == \"manual\":\n            return\n\n        if self.ipython_dir != '~/.ipython':\n            self.ipython_dir = os.path.abspath(os.path.expanduser(self.ipython_dir))\n\n        if self.log:\n            stdout = open(os.path.join(self.ipython_dir, \"{0}.controller.out\".format(self.profile)), 'w')\n            stderr = open(os.path.join(self.ipython_dir, \"{0}.controller.err\".format(self.profile)), 'w')\n        else:\n            stdout = open(os.devnull, 'w')\n            stderr = open(os.devnull, 'w')\n\n        try:\n            opts = [\n                'ipcontroller',\n                '' if self.ipython_dir == '~/.ipython' else '--ipython-dir={}'.format(self.ipython_dir),\n                self.interfaces if self.interfaces is not None else '--ip=*',\n                '' if self.profile == 'default' else '--profile={0}'.format(self.profile),\n                '--reuse' if self.reuse else '',\n                '--location={}'.format(self.public_ip) if self.public_ip else '',\n                '--port={}'.format(self.port) if self.port is not None else ''\n            ]\n            if self.port_range is not None:\n                opts += [\n                    '--HubFactory.hb={0},{1}'.format(self.hb_ping, self.hb_pong),\n                    '--HubFactory.control={0},{1}'.format(self.control_client, self.control_engine),\n                    '--HubFactory.mux={0},{1}'.format(self.mux_client, self.mux_engine),\n                    '--HubFactory.task={0},{1}'.format(self.task_client, self.task_engine)\n                ]\n            logger.debug(\"Starting ipcontroller with '{}'\".format(' '.join([str(x) for x in opts])))\n            self.proc = subprocess.Popen(opts, stdout=stdout, stderr=stderr, preexec_fn=os.setsid)\n        except FileNotFoundError:\n            msg = \"Could not find ipcontroller. Please make sure that ipyparallel is installed and available in your env\"\n            logger.error(msg)\n            raise ControllerError(msg)\n        except Exception as e:\n            msg = \"IPPController failed to start: {0}\".format(e)\n            logger.error(msg)\n            raise ControllerError(msg)", "response": "Start the ipython controller."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nspecify path to the ipcontroller - engine. json file.", "response": "def engine_file(self):\n        \"\"\"Specify path to the ipcontroller-engine.json file.\n\n        This file is stored in in the ipython_dir/profile folders.\n\n        Returns :\n              - str, File path to engine file\n        \"\"\"\n        return os.path.join(self.ipython_dir,\n                            'profile_{0}'.format(self.profile),\n                            'security/ipcontroller-engine.json')"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef client_file(self):\n        return os.path.join(self.ipython_dir,\n                            'profile_{0}'.format(self.profile),\n                            'security/ipcontroller-client.json')", "response": "Specify path to the ipcontroller - client. json file."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nterminating the controller process and its child processes.", "response": "def close(self):\n        \"\"\"Terminate the controller process and its child processes.\n\n        Args:\n              - None\n        \"\"\"\n        if self.reuse:\n            logger.debug(\"Ipcontroller not shutting down: reuse enabled\")\n            return\n\n        if self.mode == \"manual\":\n            logger.debug(\"Ipcontroller not shutting down: Manual mode\")\n            return\n\n        try:\n            pgid = os.getpgid(self.proc.pid)\n            os.killpg(pgid, signal.SIGTERM)\n            time.sleep(0.2)\n            os.killpg(pgid, signal.SIGKILL)\n            try:\n                self.proc.wait(timeout=1)\n                x = self.proc.returncode\n                if x == 0:\n                    logger.debug(\"Controller exited with {0}\".format(x))\n                else:\n                    logger.error(\"Controller exited with {0}. May require manual cleanup\".format(x))\n            except subprocess.TimeoutExpired:\n                logger.warn(\"Ipcontroller process:{0} cleanup failed. May require manual cleanup\".format(self.proc.pid))\n\n        except Exception as e:\n            logger.warn(\"Failed to kill the ipcontroller process[{0}]: {1}\".format(self.proc.pid, e))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncreate a hash of the task inputs.", "response": "def make_hash(self, task):\n        \"\"\"Create a hash of the task inputs.\n\n        This uses a serialization library borrowed from ipyparallel.\n        If this fails here, then all ipp calls are also likely to fail due to failure\n        at serialization.\n\n        Args:\n            - task (dict) : Task dictionary from dfk.tasks\n\n        Returns:\n            - hash (str) : A unique hash string\n        \"\"\"\n        # Function name TODO: Add fn body later\n        t = [serialize_object(task['func_name'])[0],\n             serialize_object(task['fn_hash'])[0],\n             serialize_object(task['args'])[0],\n             serialize_object(task['kwargs'])[0],\n             serialize_object(task['env'])[0]]\n        x = b''.join(t)\n        hashedsum = hashlib.md5(x).hexdigest()\n        return hashedsum"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nchecking the memoization of the task and returns the result of the function.", "response": "def check_memo(self, task_id, task):\n        \"\"\"Create a hash of the task and its inputs and check the lookup table for this hash.\n\n        If present, the results are returned. The result is a tuple indicating whether a memo\n        exists and the result, since a Null result is possible and could be confusing.\n        This seems like a reasonable option without relying on an cache_miss exception.\n\n        Args:\n            - task(task) : task from the dfk.tasks table\n\n        Returns:\n            Tuple of the following:\n            - present (Bool): Is this present in the memo_lookup_table\n            - Result (Py Obj): Result of the function if present in table\n\n        This call will also set task['hashsum'] to the unique hashsum for the func+inputs.\n        \"\"\"\n        if not self.memoize or not task['memoize']:\n            task['hashsum'] = None\n            return None, None\n\n        hashsum = self.make_hash(task)\n        present = False\n        result = None\n        if hashsum in self.memo_lookup_table:\n            present = True\n            result = self.memo_lookup_table[hashsum]\n            logger.info(\"Task %s using result from cache\", task_id)\n\n        task['hashsum'] = hashsum\n        return present, result"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef update_memo(self, task_id, task, r):\n        if not self.memoize or not task['memoize']:\n            return\n\n        if task['hashsum'] in self.memo_lookup_table:\n            logger.info('Updating appCache entry with latest %s:%s call' %\n                        (task['func_name'], task_id))\n            self.memo_lookup_table[task['hashsum']] = r\n        else:\n            self.memo_lookup_table[task['hashsum']] = r", "response": "Updates the memoization lookup table with the result from a task."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _nbytes(buf):\n    if isinstance(buf, memoryview):\n        if PY3:\n            # py3 introduces nbytes attribute\n            return buf.nbytes\n        else:\n            # compute nbytes on py2\n            size = buf.itemsize\n            for dim in buf.shape:\n                size *= dim\n            return size\n    else:\n        # not a memoryview, raw bytes/ py2 buffer\n        return len(buf)", "response": "Return byte - size of a memoryview or buffer."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nextracts buffers larger than a certain threshold.", "response": "def _extract_buffers(obj, threshold=MAX_BYTES):\n    \"\"\"Extract buffers larger than a certain threshold.\"\"\"\n    buffers = []\n    if isinstance(obj, CannedObject) and obj.buffers:\n        for i, buf in enumerate(obj.buffers):\n            nbytes = _nbytes(buf)\n            if nbytes > threshold:\n                # buffer larger than threshold, prevent pickling\n                obj.buffers[i] = None\n                buffers.append(buf)\n            # buffer too small for separate send, coerce to bytes\n            # because pickling buffer objects just results in broken pointers\n            elif isinstance(buf, memoryview):\n                obj.buffers[i] = buf.tobytes()\n            elif isinstance(buf, buffer):\n                obj.buffers[i] = bytes(buf)\n    return buffers"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef serialize_object(obj, buffer_threshold=MAX_BYTES, item_threshold=MAX_ITEMS):\n    buffers = []\n    if istype(obj, sequence_types) and len(obj) < item_threshold:\n        cobj = can_sequence(obj)\n        for c in cobj:\n            buffers.extend(_extract_buffers(c, buffer_threshold))\n    elif istype(obj, dict) and len(obj) < item_threshold:\n        cobj = {}\n        for k in sorted(obj):\n            c = can(obj[k])\n            buffers.extend(_extract_buffers(c, buffer_threshold))\n            cobj[k] = c\n    else:\n        cobj = can(obj)\n        buffers.extend(_extract_buffers(cobj, buffer_threshold))\n\n    buffers.insert(0, pickle.dumps(cobj, PICKLE_PROTOCOL))\n    return buffers", "response": "Serialize an object into a list of sendable buffers."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreconstructing an object serialized by serialize_object from data buffers.", "response": "def deserialize_object(buffers, g=None):\n    \"\"\"Reconstruct an object serialized by serialize_object from data buffers.\n\n    Parameters\n    ----------\n\n    bufs : list of buffers/bytes\n\n    g : globals to be used when uncanning\n\n    Returns\n    -------\n\n    (newobj, bufs) : unpacked object, and the list of remaining unused buffers.\n    \"\"\"\n    bufs = list(buffers)\n    pobj = buffer_to_bytes_py2(bufs.pop(0))\n    canned = pickle.loads(pobj)\n    if istype(canned, sequence_types) and len(canned) < MAX_ITEMS:\n        for c in canned:\n            _restore_buffers(c, bufs)\n        newobj = uncan_sequence(canned, g)\n    elif istype(canned, dict) and len(canned) < MAX_ITEMS:\n        newobj = {}\n        for k in sorted(canned):\n            c = canned[k]\n            _restore_buffers(c, bufs)\n            newobj[k] = uncan(c, g)\n    else:\n        _restore_buffers(canned, bufs)\n        newobj = uncan(canned, g)\n\n    return newobj, bufs"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef pack_apply_message(f, args, kwargs, buffer_threshold=MAX_BYTES, item_threshold=MAX_ITEMS):\n    arg_bufs = list(chain.from_iterable(\n        serialize_object(arg, buffer_threshold, item_threshold) for arg in args))\n\n    kw_keys = sorted(kwargs.keys())\n    kwarg_bufs = list(chain.from_iterable(\n        serialize_object(kwargs[key], buffer_threshold, item_threshold) for key in kw_keys))\n\n    info = dict(nargs=len(args), narg_bufs=len(arg_bufs), kw_keys=kw_keys)\n\n    msg = [pickle.dumps(can(f), PICKLE_PROTOCOL)]\n    msg.append(pickle.dumps(info, PICKLE_PROTOCOL))\n    msg.extend(arg_bufs)\n    msg.extend(kwarg_bufs)\n\n    return msg", "response": "Pack up a function args and kwargs to be sent over the wire."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef unpack_apply_message(bufs, g=None, copy=True):\n    bufs = list(bufs)  # allow us to pop\n    assert len(bufs) >= 2, \"not enough buffers!\"\n    pf = buffer_to_bytes_py2(bufs.pop(0))\n    f = uncan(pickle.loads(pf), g)\n    pinfo = buffer_to_bytes_py2(bufs.pop(0))\n    info = pickle.loads(pinfo)\n    arg_bufs, kwarg_bufs = bufs[:info['narg_bufs']], bufs[info['narg_bufs']:]\n\n    args = []\n    for i in range(info['nargs']):\n        arg, arg_bufs = deserialize_object(arg_bufs, g)\n        args.append(arg)\n    args = tuple(args)\n    assert not arg_bufs, \"Shouldn't be any arg bufs left over\"\n\n    kwargs = {}\n    for key in info['kw_keys']:\n        kwarg, kwarg_bufs = deserialize_object(kwarg_bufs, g)\n        kwargs[key] = kwarg\n    assert not kwarg_bufs, \"Shouldn't be any kwarg bufs left over\"\n\n    return f, args, kwargs", "response": "Unpack f args kwargs from buffers packed by pack_apply_message."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _write_submit_script(self, template, script_filename, job_name, configs):\n\n        try:\n            submit_script = Template(template).substitute(jobname=job_name, **configs)\n            # submit_script = Template(template).safe_substitute(jobname=job_name, **configs)\n            with open(script_filename, 'w') as f:\n                f.write(submit_script)\n\n        except KeyError as e:\n            logger.error(\"Missing keys for submit script : %s\", e)\n            raise (SchedulerMissingArgs(e.args, self.sitename))\n\n        except IOError as e:\n            logger.error(\"Failed writing to submit script: %s\", script_filename)\n            raise (ScriptPathError(script_filename, e))\n        except Exception as e:\n            print(\"Template : \", template)\n            print(\"Args : \", job_name)\n            print(\"Kwargs : \", configs)\n            logger.error(\"Uncategorized error: %s\", e)\n            raise (e)\n\n        return True", "response": "Generate and write the submit script to a file."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef status(self, job_ids):\n        if job_ids:\n            self._status()\n        return [self.resources[jid]['status'] for jid in job_ids]", "response": "Returns the status of a list of jobs identified by the job identifiers\n       "}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef status(self, job_ids):\n        '''  Get the status of a list of jobs identified by their ids.\n\n        Args:\n            - job_ids (List of ids) : List of identifiers for the jobs\n\n        Returns:\n            - List of status codes.\n\n        '''\n\n        logger.debug(\"Checking status of: {0}\".format(job_ids))\n        for job_id in self.resources:\n\n            if self.resources[job_id]['proc']:\n\n                poll_code = self.resources[job_id]['proc'].poll()\n                if self.resources[job_id]['status'] in ['COMPLETED', 'FAILED']:\n                    continue\n\n                if poll_code is None:\n                    self.resources[job_id]['status'] = 'RUNNING'\n                elif poll_code == 0:\n                    self.resources[job_id]['status'] = 'COMPLETED'\n                elif poll_code != 0:\n                    self.resources[job_id]['status'] = 'FAILED'\n                else:\n                    logger.error(\"Internal consistency error: unexpected case in local provider state machine\")\n\n            elif self.resources[job_id]['remote_pid']:\n\n                retcode, stdout, stderr = self.channel.execute_wait('ps -p {} &> /dev/null; echo \"STATUS:$?\" ',\n                                                                    self.cmd_timeout)\n                for line in stdout.split('\\n'):\n                    if line.startswith(\"STATUS:\"):\n                        status = line.split(\"STATUS:\")[1].strip()\n                        if status == \"0\":\n                            self.resources[job_id]['status'] = 'RUNNING'\n                        else:\n                            self.resources[job_id]['status'] = 'FAILED'\n\n        return [self.resources[jid]['status'] for jid in job_ids]", "response": "Get the status of a list of jobs identified by their ids."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nsubmitting a command onto a Local Resource Manager job of blocksize parallel elements.", "response": "def submit(self, command, blocksize, tasks_per_node, job_name=\"parsl.auto\"):\n        ''' Submits the command onto an Local Resource Manager job of blocksize parallel elements.\n        Submit returns an ID that corresponds to the task that was just submitted.\n\n        If tasks_per_node <  1:\n             1/tasks_per_node is provisioned\n\n        If tasks_per_node == 1:\n             A single node is provisioned\n\n        If tasks_per_node >  1 :\n             tasks_per_node * blocksize number of nodes are provisioned.\n\n        Args:\n             - command  :(String) Commandline invocation to be made on the remote side.\n             - blocksize   :(float) - Not really used for local\n             - tasks_per_node (int) : command invocations to be launched per node\n\n        Kwargs:\n             - job_name (String): Name for job, must be unique\n\n        Returns:\n             - None: At capacity, cannot provision more\n             - job_id: (string) Identifier for the job\n\n        '''\n\n        job_name = \"{0}.{1}\".format(job_name, time.time())\n\n        # Set script path\n        script_path = \"{0}/{1}.sh\".format(self.script_dir, job_name)\n        script_path = os.path.abspath(script_path)\n\n        wrap_command = self.worker_init + '\\n' + self.launcher(command, tasks_per_node, self.nodes_per_block)\n\n        self._write_submit_script(wrap_command, script_path)\n\n        job_id = None\n        proc = None\n        remote_pid = None\n        if (self.move_files is None and not isinstance(self.channel, LocalChannel)) or (self.move_files):\n            logger.debug(\"Moving start script\")\n            script_path = self.channel.push_file(script_path, self.channel.script_dir)\n\n        if not isinstance(self.channel, LocalChannel):\n            logger.debug(\"Launching in remote mode\")\n            # Bash would return until the streams are closed. So we redirect to a outs file\n            cmd = 'bash {0} &> {0}.out & \\n echo \"PID:$!\" '.format(script_path)\n            retcode, stdout, stderr = self.channel.execute_wait(cmd, self.cmd_timeout)\n            for line in stdout.split('\\n'):\n                if line.startswith(\"PID:\"):\n                    remote_pid = line.split(\"PID:\")[1].strip()\n                    job_id = remote_pid\n            if job_id is None:\n                logger.warning(\"Channel failed to start remote command/retrieve PID\")\n        else:\n\n            try:\n                job_id, proc = self.channel.execute_no_wait('bash {0}'.format(script_path), self.cmd_timeout)\n            except Exception as e:\n                logger.debug(\"Channel execute failed for: {}, {}\".format(self.channel, e))\n                raise\n\n        self.resources[job_id] = {'job_id': job_id, 'status': 'RUNNING',\n                                  'blocksize': blocksize,\n                                  'remote_pid': remote_pid,\n                                  'proc': proc}\n\n        return job_id"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef cancel(self, job_ids):\n        ''' Cancels the jobs specified by a list of job ids\n\n        Args:\n        job_ids : [<job_id> ...]\n\n        Returns :\n        [True/False...] : If the cancel operation fails the entire list will be False.\n        '''\n        for job in job_ids:\n            logger.debug(\"Terminating job/proc_id: {0}\".format(job))\n            # Here we are assuming that for local, the job_ids are the process id's\n            if self.resources[job]['proc']:\n                proc = self.resources[job]['proc']\n                os.killpg(os.getpgid(proc.pid), signal.SIGTERM)\n                self.resources[job]['status'] = 'CANCELLED'\n\n            elif self.resources[job]['remote_pid']:\n                cmd = \"kill -- -$(ps -o pgid={} | grep -o '[0-9]*')\".format(self.resources[job]['remote_pid'])\n                retcode, stdout, stderr = self.channel.execute_wait(cmd, self.cmd_timeout)\n                if retcode != 0:\n                    logger.warning(\"Failed to kill PID: {} and child processes on {}\".format(self.resources[job]['remote_pid'],\n                                                                                             self.label))\n\n        rets = [True for i in job_ids]\n        return rets", "response": "Cancels the jobs specified by a list of job ids."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsubmits a command onto a Local Resource Manager job.", "response": "def submit(self, command, blocksize, tasks_per_node, job_name=\"parsl.auto\"):\n        \"\"\" Submits the command onto an Local Resource Manager job of blocksize parallel elements.\n        Submit returns an ID that corresponds to the task that was just submitted.\n\n        If tasks_per_node <  1 : ! This is illegal. tasks_per_node should be integer\n\n        If tasks_per_node == 1:\n             A single node is provisioned\n\n        If tasks_per_node >  1 :\n             tasks_per_node * blocksize number of nodes are provisioned.\n\n        Args:\n             - command  :(String) Commandline invocation to be made on the remote side.\n             - blocksize   :(float)\n             - tasks_per_node (int) : command invocations to be launched per node\n\n        Kwargs:\n             - job_name (String): Name for job, must be unique\n\n        Returns:\n             - None: At capacity, cannot provision more\n             - job_id: (string) Identifier for the job\n\n        \"\"\"\n\n        if self.provisioned_blocks >= self.max_blocks:\n            logger.warn(\"[%s] at capacity, cannot add more blocks now\", self.label)\n            return None\n\n        # Note: Fix this later to avoid confusing behavior.\n        # We should always allocate blocks in integer counts of node_granularity\n        if blocksize < self.nodes_per_block:\n            blocksize = self.nodes_per_block\n\n        account_opt = '-A {}'.format(self.account) if self.account is not None else ''\n\n        job_name = \"parsl.{0}.{1}\".format(job_name, time.time())\n\n        script_path = \"{0}/{1}.submit\".format(self.script_dir, job_name)\n        script_path = os.path.abspath(script_path)\n\n        job_config = {}\n        job_config[\"scheduler_options\"] = self.scheduler_options\n        job_config[\"worker_init\"] = self.worker_init\n\n        logger.debug(\"Requesting blocksize:%s nodes_per_block:%s tasks_per_node:%s\",\n                     blocksize, self.nodes_per_block, tasks_per_node)\n\n        # Wrap the command\n        job_config[\"user_script\"] = self.launcher(command, tasks_per_node, self.nodes_per_block)\n\n        queue_opt = '-q {}'.format(self.queue) if self.queue is not None else ''\n\n        logger.debug(\"Writing submit script\")\n        self._write_submit_script(template_string, script_path, job_name, job_config)\n\n        channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)\n\n        command = 'qsub -n {0} {1} -t {2} {3} {4}'.format(\n            self.nodes_per_block, queue_opt, wtime_to_minutes(self.walltime), account_opt, channel_script_path)\n        logger.debug(\"Executing {}\".format(command))\n\n        retcode, stdout, stderr = super().execute_wait(command)\n\n        # TODO : FIX this block\n        if retcode != 0:\n            logger.error(\"Failed command: {0}\".format(command))\n            logger.error(\"Launch failed stdout:\\n{0} \\nstderr:{1}\\n\".format(stdout, stderr))\n\n        logger.debug(\"Retcode:%s STDOUT:%s STDERR:%s\", retcode, stdout.strip(), stderr.strip())\n\n        job_id = None\n\n        if retcode == 0:\n            # We should be getting only one line back\n            job_id = stdout.strip()\n            self.resources[job_id] = {'job_id': job_id, 'status': 'PENDING', 'blocksize': blocksize}\n        else:\n            logger.error(\"Submission of command to scale_out failed: {0}\".format(stderr))\n            raise (ScaleOutFailed(self.__class__, \"Request to submit job to local scheduler failed\"))\n\n        logger.debug(\"Returning job id : {0}\".format(job_id))\n        return job_id"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef initialize_boto_client(self):\n\n        self.session = self.create_session()\n        self.client = self.session.client('ec2')\n        self.ec2 = self.session.resource('ec2')\n        self.instances = []\n        self.instance_states = {}\n        self.vpc_id = 0\n        self.sg_id = 0\n        self.sn_ids = []", "response": "Initialize the boto client."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef read_state_file(self, state_file):\n        try:\n            fh = open(state_file, 'r')\n            state = json.load(fh)\n            self.vpc_id = state['vpcID']\n            self.sg_id = state['sgID']\n            self.sn_ids = state['snIDs']\n            self.instances = state['instances']\n        except Exception as e:\n            logger.debug(\"Caught exception while reading state file: {0}\".format(e))\n            raise e\n        logger.debug(\"Done reading state from the local state file.\")", "response": "Read the state file and store the state information."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef write_state_file(self):\n        fh = open('awsproviderstate.json', 'w')\n        state = {}\n        state['vpcID'] = self.vpc_id\n        state['sgID'] = self.sg_id\n        state['snIDs'] = self.sn_ids\n        state['instances'] = self.instances\n        state[\"instanceState\"] = self.instance_states\n        fh.write(json.dumps(state, indent=4))", "response": "Save information that must persist to a file."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates a session for the current user.", "response": "def create_session(self):\n        \"\"\"Create a session.\n\n        First we look in self.key_file for a path to a json file with the\n        credentials. The key file should have 'AWSAccessKeyId' and 'AWSSecretKey'.\n\n        Next we look at self.profile for a profile name and try\n        to use the Session call to automatically pick up the keys for the profile from\n        the user default keys file ~/.aws/config.\n\n        Finally, boto3 will look for the keys in environment variables:\n        AWS_ACCESS_KEY_ID: The access key for your AWS account.\n        AWS_SECRET_ACCESS_KEY: The secret key for your AWS account.\n        AWS_SESSION_TOKEN: The session key for your AWS account.\n        This is only needed when you are using temporary credentials.\n        The AWS_SECURITY_TOKEN environment variable can also be used,\n        but is only supported for backwards compatibility purposes.\n        AWS_SESSION_TOKEN is supported by multiple AWS SDKs besides python.\n        \"\"\"\n\n        session = None\n\n        if self.key_file is not None:\n            credfile = os.path.expandvars(os.path.expanduser(self.key_file))\n\n            try:\n                with open(credfile, 'r') as f:\n                    creds = json.load(f)\n            except json.JSONDecodeError as e:\n                logger.error(\n                    \"EC2Provider '{}': json decode error in credential file {}\".format(self.label, credfile)\n                )\n                raise e\n\n            except Exception as e:\n                logger.debug(\n                    \"EC2Provider '{0}' caught exception while reading credential file: {1}\".format(\n                        self.label, credfile\n                    )\n                )\n                raise e\n\n            logger.debug(\"EC2Provider '{}': Using credential file to create session\".format(self.label))\n            session = boto3.session.Session(region_name=self.region, **creds)\n        elif self.profile is not None:\n            logger.debug(\"EC2Provider '{}': Using profile name to create session\".format(self.label))\n            session = boto3.session.Session(\n                profile_name=self.profile, region_name=self.region\n            )\n        else:\n            logger.debug(\"EC2Provider '{}': Using environment variables to create session\".format(self.label))\n            session = boto3.session.Session(region_name=self.region)\n\n        return session"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef create_vpc(self):\n\n        try:\n            # We use a large VPC so that the cluster can get large\n            vpc = self.ec2.create_vpc(\n                CidrBlock='10.0.0.0/16',\n                AmazonProvidedIpv6CidrBlock=False,\n            )\n        except Exception as e:\n            # This failure will cause a full abort\n            logger.error(\"{}\\n\".format(e))\n            raise e\n\n        # Attach internet gateway so that our cluster can\n        # talk to the outside internet\n        internet_gateway = self.ec2.create_internet_gateway()\n        internet_gateway.attach_to_vpc(VpcId=vpc.vpc_id)  # Returns None\n        self.internet_gateway = internet_gateway.id\n\n        # Create and configure route table to allow proper traffic\n        route_table = self.config_route_table(vpc, internet_gateway)\n        self.route_table = route_table.id\n\n        # Get all avaliability zones\n        availability_zones = self.client.describe_availability_zones()\n\n        # go through AZs and set up a subnet per\n        for num, zone in enumerate(availability_zones['AvailabilityZones']):\n            if zone['State'] == \"available\":\n\n                # Create a large subnet (4000 max nodes)\n                subnet = vpc.create_subnet(\n                    CidrBlock='10.0.{}.0/20'.format(16 * num), AvailabilityZone=zone['ZoneName']\n                )\n\n                # Make subnet accessible\n                subnet.meta.client.modify_subnet_attribute(\n                    SubnetId=subnet.id, MapPublicIpOnLaunch={\"Value\": True}\n                )\n\n                route_table.associate_with_subnet(SubnetId=subnet.id)\n                self.sn_ids.append(subnet.id)\n            else:\n                logger.info(\"{} unavailable\".format(zone['ZoneName']))\n        # Security groups\n        self.security_group(vpc)\n        self.vpc_id = vpc.id\n        return vpc", "response": "Create and configure a new VPC for the current cluster and return the ID of the VPC."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef security_group(self, vpc):\n\n        sg = vpc.create_security_group(\n            GroupName=\"private-subnet\", Description=\"security group for remote executors\"\n        )\n\n        ip_ranges = [{'CidrIp': '10.0.0.0/16'}]\n\n        # Allows all ICMP in, all TCP and UDP in within VPC\n        in_permissions = [\n            {\n                'IpProtocol': 'TCP',\n                'FromPort': 0,\n                'ToPort': 65535,\n                'IpRanges': ip_ranges,\n            }, {\n                'IpProtocol': 'UDP',\n                'FromPort': 0,\n                'ToPort': 65535,\n                'IpRanges': ip_ranges,\n            }, {\n                'IpProtocol': 'ICMP',\n                'FromPort': -1,\n                'ToPort': -1,\n                'IpRanges': [{\n                    'CidrIp': '0.0.0.0/0'\n                }],\n            }, {\n                'IpProtocol': 'TCP',\n                'FromPort': 22,\n                'ToPort': 22,\n                'IpRanges': [{\n                    'CidrIp': '0.0.0.0/0'\n                }],\n            }\n        ]\n\n        # Allows all TCP out, all TCP and UDP out within VPC\n        out_permissions = [\n            {\n                'IpProtocol': 'TCP',\n                'FromPort': 0,\n                'ToPort': 65535,\n                'IpRanges': [{\n                    'CidrIp': '0.0.0.0/0'\n                }],\n            },\n            {\n                'IpProtocol': 'TCP',\n                'FromPort': 0,\n                'ToPort': 65535,\n                'IpRanges': ip_ranges,\n            },\n            {\n                'IpProtocol': 'UDP',\n                'FromPort': 0,\n                'ToPort': 65535,\n                'IpRanges': ip_ranges,\n            },\n        ]\n\n        sg.authorize_ingress(IpPermissions=in_permissions)\n        sg.authorize_egress(IpPermissions=out_permissions)\n        self.sg_id = sg.id\n        return sg", "response": "Create and configure a security group for the instance of the current user."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nconfigures the route table for the given VPC.", "response": "def config_route_table(self, vpc, internet_gateway):\n        \"\"\"Configure route table for Virtual Private Cloud (VPC).\n\n        Parameters\n        ----------\n        vpc : dict\n            Representation of the VPC (created by create_vpc()).\n        internet_gateway : dict\n            Representation of the internet gateway (created by create_vpc()).\n        \"\"\"\n        route_table = vpc.create_route_table()\n        route_table.create_route(\n            DestinationCidrBlock='0.0.0.0/0', GatewayId=internet_gateway.internet_gateway_id\n        )\n        return route_table"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef spin_up_instance(self, command, job_name):\n\n        command = Template(template_string).substitute(jobname=job_name,\n                                                       user_script=command,\n                                                       linger=str(self.linger).lower(),\n                                                       worker_init=self.worker_init)\n        instance_type = self.instance_type\n        subnet = self.sn_ids[0]\n        ami_id = self.image_id\n        total_instances = len(self.instances)\n\n        if float(self.spot_max_bid) > 0:\n            spot_options = {\n                'MarketType': 'spot',\n                'SpotOptions': {\n                    'MaxPrice': str(self.spot_max_bid),\n                    'SpotInstanceType': 'one-time',\n                    'InstanceInterruptionBehavior': 'terminate'\n                }\n            }\n        else:\n            spot_options = {}\n\n        if total_instances > self.max_nodes:\n            logger.warn(\"Exceeded instance limit ({}). Cannot continue\\n\".format(self.max_nodes))\n            return [None]\n        try:\n            tag_spec = [{\"ResourceType\": \"instance\", \"Tags\": [{'Key': 'Name', 'Value': job_name}]}]\n\n            instance = self.ec2.create_instances(\n                MinCount=1,\n                MaxCount=1,\n                InstanceType=instance_type,\n                ImageId=ami_id,\n                KeyName=self.key_name,\n                SubnetId=subnet,\n                SecurityGroupIds=[self.sg_id],\n                TagSpecifications=tag_spec,\n                InstanceMarketOptions=spot_options,\n                InstanceInitiatedShutdownBehavior='terminate',\n                IamInstanceProfile={'Arn': self.iam_instance_profile_arn},\n                UserData=command\n            )\n        except ClientError as e:\n            print(e)\n            logger.error(e.response)\n            return [None]\n\n        except Exception as e:\n            logger.error(\"Request for EC2 resources failed : {0}\".format(e))\n            return [None]\n\n        self.instances.append(instance[0].id)\n        logger.info(\n            \"Started up 1 instance {} . Instance type:{}\".format(instance[0].id, instance_type)\n        )\n        return instance", "response": "Start an instance in the VPC in the first available subnet."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nshutting down a single instance.", "response": "def shut_down_instance(self, instances=None):\n        \"\"\"Shut down a list of instances, if provided.\n\n        If no instance is provided, the last instance started up will be shut down.\n        \"\"\"\n\n        if instances and len(self.instances) > 0:\n            print(instances)\n            try:\n                print([i.id for i in instances])\n            except Exception as e:\n                print(e)\n            term = self.client.terminate_instances(InstanceIds=instances)\n            logger.info(\"Shut down {} instances (ids:{}\".format(len(instances), str(instances)))\n        elif len(self.instances) > 0:\n            instance = self.instances.pop()\n            term = self.client.terminate_instances(InstanceIds=[instance])\n            logger.info(\"Shut down 1 instance (id:{})\".format(instance))\n        else:\n            logger.warn(\"No Instances to shut down.\\n\")\n            return -1\n        self.get_instance_state()\n        return term"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef get_instance_state(self, instances=None):\n        if instances:\n            desc = self.client.describe_instances(InstanceIds=instances)\n        else:\n            desc = self.client.describe_instances(InstanceIds=self.instances)\n        # pprint.pprint(desc['Reservations'],indent=4)\n        for i in range(len(desc['Reservations'])):\n            instance = desc['Reservations'][i]['Instances'][0]\n            self.instance_states[instance['InstanceId']] = instance['State']['Name']\n        return self.instance_states", "response": "Get states of all instances on EC2 which were started by this file."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef status(self, job_ids):\n\n        all_states = []\n\n        status = self.client.describe_instances(InstanceIds=job_ids)\n        for r in status['Reservations']:\n            for i in r['Instances']:\n                instance_id = i['InstanceId']\n                instance_state = translate_table.get(i['State']['Name'], 'UNKNOWN')\n                self.resources[instance_id]['status'] = instance_state\n                all_states.extend([instance_state])\n\n        return all_states", "response": "Get the status of a list of jobs identified by their ids."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef submit(self, command='sleep 1', blocksize=1, tasks_per_node=1, job_name=\"parsl.auto\"):\n\n        job_name = \"parsl.auto.{0}\".format(time.time())\n        wrapped_cmd = self.launcher(command,\n                                    tasks_per_node,\n                                    self.nodes_per_block)\n        [instance, *rest] = self.spin_up_instance(command=wrapped_cmd, job_name=job_name)\n\n        if not instance:\n            logger.error(\"Failed to submit request to EC2\")\n            return None\n\n        logger.debug(\"Started instance_id: {0}\".format(instance.instance_id))\n\n        state = translate_table.get(instance.state['Name'], \"PENDING\")\n\n        self.resources[instance.instance_id] = {\n            \"job_id\": instance.instance_id,\n            \"instance\": instance,\n            \"status\": state\n        }\n\n        return instance.instance_id", "response": "Submit the command onto a freshly instantiated AWS EC2 instance."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncancels the jobs specified by a list of job ids.", "response": "def cancel(self, job_ids):\n        \"\"\"Cancel the jobs specified by a list of job ids.\n\n        Parameters\n        ----------\n        job_ids : list of str\n            List of of job identifiers\n\n        Returns\n        -------\n        list of bool\n            Each entry in the list will contain False if the operation fails. Otherwise, the entry will be True.\n        \"\"\"\n\n        if self.linger is True:\n            logger.debug(\"Ignoring cancel requests due to linger mode\")\n            return [False for x in job_ids]\n\n        try:\n            self.client.terminate_instances(InstanceIds=list(job_ids))\n        except Exception as e:\n            logger.error(\"Caught error while attempting to remove instances: {0}\".format(job_ids))\n            raise e\n        else:\n            logger.debug(\"Removed the instances: {0}\".format(job_ids))\n\n        for job_id in job_ids:\n            self.resources[job_id][\"status\"] = \"COMPLETED\"\n\n        for job_id in job_ids:\n            self.instances.remove(job_id)\n\n        return [True for x in job_ids]"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef show_summary(self):\n        self.get_instance_state()\n        status_string = \"EC2 Summary:\\n\\tVPC IDs: {}\\n\\tSubnet IDs: \\\n{}\\n\\tSecurity Group ID: {}\\n\\tRunning Instance IDs: {}\\n\".format(\n            self.vpc_id, self.sn_ids, self.sg_id, self.instances\n        )\n        status_string += \"\\tInstance States:\\n\\t\\t\"\n        self.get_instance_state()\n        for state in self.instance_states.keys():\n            status_string += \"Instance ID: {}  State: {}\\n\\t\\t\".format(\n                state, self.instance_states[state]\n            )\n        status_string += \"\\n\"\n        logger.info(status_string)\n        return status_string", "response": "Print human readable summary of current state to log and to console."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nscale out the existing resources.", "response": "def scale_out(self, blocks=1, block_size=1):\n        ''' Scale out the existing resources.\n        '''\n        self.config['sites.jetstream.{0}'.format(self.pool)]['flavor']\n        count = 0\n        if blocks == 1:\n            block_id = len(self.blocks)\n            self.blocks[block_id] = []\n            for instance_id in range(0, block_size):\n                instances = self.server_manager.create(\n                    'parsl-{0}-{1}'.format(block_id, instance_id),  # Name\n                    self.client.images.get('87e08a17-eae2-4ce4-9051-c561d9a54bde'),  # Image_id\n                    self.client.flavors.list()[0],\n                    min_count=1,\n                    max_count=1,\n                    userdata=setup_script.format(engine_config=self.engine_config),\n                    key_name='TG-MCB090174-api-key',\n                    security_groups=['global-ssh'],\n                    nics=[{\n                        \"net-id\": '724a50cf-7f11-4b3b-a884-cd7e6850e39e',\n                        \"net-name\": 'PARSL-priv-net',\n                        \"v4-fixed-ip\": ''\n                    }])\n                self.blocks[block_id].extend([instances])\n                count += 1\n\n        return count"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef execute_task(f, args, kwargs, user_ns):\n    fname = getattr(f, '__name__', 'f')\n    prefix = \"parsl_\"\n    fname = prefix + \"f\"\n    argname = prefix + \"args\"\n    kwargname = prefix + \"kwargs\"\n    resultname = prefix + \"result\"\n\n    user_ns.update({fname: f,\n                    argname: args,\n                    kwargname: kwargs,\n                    resultname: resultname})\n\n    code = \"{0} = {1}(*{2}, **{3})\".format(resultname, fname,\n                                           argname, kwargname)\n    try:\n        exec(code, user_ns, user_ns)\n\n    except Exception as e:\n        logger.warning(\"Caught exception; will raise it: {}\".format(e))\n        raise e\n\n    else:\n        return user_ns.get(resultname)", "response": "Deserialize the buffer and execute the task."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nstart a file logger.", "response": "def start_file_logger(filename, rank, name='parsl', level=logging.DEBUG, format_string=None):\n    \"\"\"Add a stream log handler.\n\n    Args:\n        - filename (string): Name of the file to write logs to\n        - name (string): Logger name\n        - level (logging.LEVEL): Set the logging level.\n        - format_string (string): Set the format string\n\n    Returns:\n       -  None\n    \"\"\"\n\n    try:\n        os.makedirs(os.path.dirname(filename), 511, True)\n    except Exception as e:\n        print(\"Caught exception with trying to make log dirs: {}\".format(e))\n\n    if format_string is None:\n        format_string = \"%(asctime)s %(name)s:%(lineno)d Rank:{0} [%(levelname)s]  %(message)s\".format(\n            rank)\n    global logger\n    logger = logging.getLogger(name)\n    logger.setLevel(logging.DEBUG)\n    handler = logging.FileHandler(filename)\n    handler.setLevel(level)\n    formatter = logging.Formatter(format_string, datefmt='%Y-%m-%d %H:%M:%S')\n    handler.setFormatter(formatter)\n    logger.addHandler(handler)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef worker(worker_id, task_url, debug=True, logdir=\"workers\", uid=\"1\"):\n\n    start_file_logger('{}/{}/worker_{}.log'.format(logdir, uid, worker_id),\n                      0,\n                      level=logging.DEBUG if debug is True else logging.INFO)\n\n    logger.info(\"Starting worker {}\".format(worker_id))\n\n    task_ids_received = []\n\n    message_q = zmq_pipes.WorkerMessages(task_url)\n\n    while True:\n        print(\"Worker loop iteration starting\")\n        task_id, buf = message_q.get()\n        task_ids_received.append(task_id)\n\n        user_ns = locals()\n        user_ns.update({'__builtins__': __builtins__})\n        f, args, kwargs = unpack_apply_message(buf, user_ns, copy=False)\n\n        logger.debug(\"Worker {} received task {}\".format(worker_id, task_id))\n        result = execute_task(f, args, kwargs, user_ns)\n        logger.debug(\"Worker {} completed task {}\".format(worker_id, task_id))\n\n        reply = {\"result\": result, \"worker_id\": worker_id}\n        message_q.put(task_id, serialize_object(reply))\n        logger.debug(\"Result sent\")", "response": "This function is used to run a worker task."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nupdate the resource dictionary with job statuses.", "response": "def _status(self):\n        \"\"\"Update the resource dictionary with job statuses.\"\"\"\n\n        job_id_list = ' '.join(self.resources.keys())\n        cmd = \"condor_q {0} -af:jr JobStatus\".format(job_id_list)\n        retcode, stdout, stderr = super().execute_wait(cmd)\n        \"\"\"\n        Example output:\n\n        $ condor_q 34524642.0 34524643.0 -af:jr JobStatus\n        34524642.0 2\n        34524643.0 1\n        \"\"\"\n\n        for line in stdout.strip().split('\\n'):\n            parts = line.split()\n            job_id = parts[0]\n            status = translate_table.get(parts[1], 'UNKNOWN')\n            self.resources[job_id]['status'] = status"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsubmitting a command onto a Local Resource Manager job of blocksize parallel elements.", "response": "def submit(self, command, blocksize, tasks_per_node, job_name=\"parsl.auto\"):\n        \"\"\"Submits the command onto an Local Resource Manager job of blocksize parallel elements.\n\n        example file with the complex case of multiple submits per job:\n            Universe =vanilla\n            output = out.$(Cluster).$(Process)\n            error = err.$(Cluster).$(Process)\n            log = log.$(Cluster)\n            leave_in_queue = true\n            executable = test.sh\n            queue 5\n            executable = foo\n            queue 1\n\n        $ condor_submit test.sub\n        Submitting job(s)......\n        5 job(s) submitted to cluster 118907.\n        1 job(s) submitted to cluster 118908.\n\n        Parameters\n        ----------\n        command : str\n            Command to execute\n        blocksize : int\n            Number of blocks to request.\n        job_name : str\n            Job name prefix.\n        tasks_per_node : int\n            command invocations to be launched per node\n        Returns\n        -------\n        None or str\n            None if at capacity and cannot provision more; otherwise the identifier for the job.\n        \"\"\"\n\n        logger.debug(\"Attempting to launch with blocksize: {}\".format(blocksize))\n        if self.provisioned_blocks >= self.max_blocks:\n            template = \"Provider {} is currently using {} blocks while max_blocks is {}; no blocks will be added\"\n            logger.warn(template.format(self.label, self.provisioned_blocks, self.max_blocks))\n            return None\n\n        # Note: Fix this later to avoid confusing behavior.\n        # We should always allocate blocks in integer counts of node_granularity\n        blocksize = max(self.nodes_per_block, blocksize)\n\n        job_name = \"parsl.{0}.{1}\".format(job_name, time.time())\n\n        script_path = \"{0}/{1}.submit\".format(self.script_dir, job_name)\n        script_path = os.path.abspath(script_path)\n        userscript_path = \"{0}/{1}.script\".format(self.script_dir, job_name)\n        userscript_path = os.path.abspath(userscript_path)\n\n        self.environment[\"JOBNAME\"] = \"'{}'\".format(job_name)\n\n        job_config = {}\n        job_config[\"job_name\"] = job_name\n        job_config[\"submit_script_dir\"] = self.channel.script_dir\n        job_config[\"project\"] = self.project\n        job_config[\"nodes\"] = self.nodes_per_block\n        job_config[\"scheduler_options\"] = self.scheduler_options\n        job_config[\"worker_init\"] = self.worker_init\n        job_config[\"user_script\"] = command\n        job_config[\"tasks_per_node\"] = tasks_per_node\n        job_config[\"requirements\"] = self.requirements\n        job_config[\"environment\"] = ' '.join(['{}={}'.format(key, value) for key, value in self.environment.items()])\n\n        # Move the user script\n        # This is where the command should be wrapped by the launchers.\n        wrapped_command = self.launcher(command,\n                                        tasks_per_node,\n                                        self.nodes_per_block)\n\n        with open(userscript_path, 'w') as f:\n            f.write(job_config[\"worker_init\"] + '\\n' + wrapped_command)\n\n        user_script_path = self.channel.push_file(userscript_path, self.channel.script_dir)\n        the_input_files = [user_script_path] + self.transfer_input_files\n        job_config[\"input_files\"] = ','.join(the_input_files)\n        job_config[\"job_script\"] = os.path.basename(user_script_path)\n\n        # Construct and move the submit script\n        self._write_submit_script(template_string, script_path, job_name, job_config)\n        channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)\n\n        cmd = \"condor_submit {0}\".format(channel_script_path)\n        retcode, stdout, stderr = super().execute_wait(cmd, 30)\n        logger.debug(\"Retcode:%s STDOUT:%s STDERR:%s\", retcode, stdout.strip(), stderr.strip())\n\n        job_id = []\n\n        if retcode == 0:\n            for line in stdout.split('\\n'):\n                if re.match('^[0-9]', line) is not None:\n                    cluster = line.split(\" \")[5]\n                    # We know the first job id (\"process\" in condor terms) within a\n                    # cluster is 0 and we know the total number of jobs from\n                    # condor_submit, so we use some list comprehensions to expand\n                    # the condor_submit output into job IDs\n                    # e.g., ['118907.0', '118907.1', '118907.2', '118907.3', '118907.4', '118908.0']\n                    processes = [str(x) for x in range(0, int(line[0]))]\n                    job_id += [cluster + process for process in processes]\n\n            self._add_resource(job_id)\n        return job_id[0]"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncompose the engine launch command.", "response": "def compose_containerized_launch_cmd(self, filepath, engine_dir, container_image):\n        \"\"\"Reads the json contents from filepath and uses that to compose the engine launch command.\n\n        Notes: Add this to the ipengine launch for debug logs :\n                          --log-to-file --debug\n        Args:\n            filepath (str): Path to the engine file\n            engine_dir (str): CWD for the engines .\n            container_image (str): The container to be used to launch workers\n        \"\"\"\n        self.engine_file = os.path.expanduser(filepath)\n        uid = str(uuid.uuid4())\n        engine_json = None\n        try:\n            with open(self.engine_file, 'r') as f:\n                engine_json = f.read()\n\n        except OSError as e:\n            logger.error(\"Could not open engine_json : \", self.engine_file)\n            raise e\n\n        return \"\"\"mkdir -p {0}\ncd {0}\ncat <<EOF > ipengine.{uid}.json\n{1}\nEOF\n\nDOCKER_ID=$(docker create --network host {2} ipengine --file=/tmp/ipengine.{uid}.json) {debug_option}\ndocker cp ipengine.{uid}.json $DOCKER_ID:/tmp/ipengine.{uid}.json\n\n# Copy current dir to the working directory\nDOCKER_CWD=$(docker image inspect --format='{{{{.Config.WorkingDir}}}}' {2})\ndocker cp -a . $DOCKER_ID:$DOCKER_CWD\ndocker start $DOCKER_ID\n\nat_exit() {{\n  echo \"Caught SIGTERM/SIGINT signal!\"\n  docker stop $DOCKER_ID\n}}\n\ntrap at_exit SIGTERM SIGINT\nsleep infinity\n\"\"\".format(engine_dir, engine_json, container_image, debug_option=self.debug_option, uid=uid)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef scale_out(self, blocks=1):\n        r = []\n        for i in range(blocks):\n            if self.provider:\n                block = self.provider.submit(self.launch_cmd, 1, self.workers_per_node)\n                logger.debug(\"Launched block {}:{}\".format(i, block))\n                if not block:\n                    raise(ScalingFailed(self.provider.label,\n                                        \"Attempts to provision nodes via provider has failed\"))\n                self.engines.extend([block])\n                r.extend([block])\n        else:\n            logger.error(\"No execution provider available\")\n            r = None\n\n        return r", "response": "Scales out the number of active workers by 1."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nscale in the number of active blocks by the specified number.", "response": "def scale_in(self, blocks):\n        \"\"\"Scale in the number of active blocks by the specified number.\n\n        \"\"\"\n        status = dict(zip(self.engines, self.provider.status(self.engines)))\n\n        # This works for blocks=0\n        to_kill = [engine for engine in status if status[engine] == \"RUNNING\"][:blocks]\n\n        if self.provider:\n            r = self.provider.cancel(to_kill)\n        else:\n            logger.error(\"No execution provider available\")\n            r = None\n\n        return r"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef status(self):\n        if self.provider:\n            status = self.provider.status(self.engines)\n\n        else:\n            status = []\n\n        return status", "response": "Returns the status of the executor via probing the execution providers."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef shutdown(self, hub=True, targets='all', block=False):\n        if self.controller:\n            logger.debug(\"IPP:Shutdown sequence: Attempting controller kill\")\n            self.controller.close()\n\n        # We do not actually do executor.shutdown because\n        # this blocks even when requested to not block, killing the\n        # controller is more effective although impolite.\n        # x = self.executor.shutdown(targets=targets,\n        #                           hub=hub,\n        #                           block=block)\n\n        logger.debug(\"Done with executor shutdown\")\n        return True", "response": "Shut down the executor including all workers and controllers."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nadd a callback to the parent to update the state.", "response": "def update_parent(self, fut):\n        \"\"\"Add a callback to the parent to update the state.\n\n        This handles the case where the user has called result on the AppFuture\n        before the parent exists.\n        \"\"\"\n        self.parent = fut\n\n        try:\n            fut.add_done_callback(self.parent_callback)\n        except Exception as e:\n            logger.error(\"add_done_callback got an exception {} which will be ignored\".format(e))"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nexecute the bash app type function and return the command line string.", "response": "def remote_side_bash_executor(func, *args, **kwargs):\n    \"\"\"Execute the bash app type function and return the command line string.\n\n    This string is reformatted with the *args, and **kwargs\n    from call time.\n    \"\"\"\n    import os\n    import time\n    import subprocess\n    import logging\n    import parsl.app.errors as pe\n\n    logging.basicConfig(filename='/tmp/bashexec.{0}.log'.format(time.time()), level=logging.DEBUG)\n\n    # start_t = time.time()\n\n    func_name = func.__name__\n\n    partial_cmdline = None\n\n    # Try to run the func to compose the commandline\n    try:\n        # Execute the func to get the commandline\n        partial_cmdline = func(*args, **kwargs)\n        # Reformat the commandline with current args and kwargs\n        executable = partial_cmdline.format(*args, **kwargs)\n\n    except AttributeError as e:\n        if partial_cmdline is not None:\n            raise pe.AppBadFormatting(\"App formatting failed for app '{}' with AttributeError: {}\".format(func_name, e))\n        else:\n            raise pe.BashAppNoReturn(\"Bash app '{}' did not return a value, or returned none - with this exception: {}\".format(func_name, e), None)\n\n    except IndexError as e:\n        raise pe.AppBadFormatting(\"App formatting failed for app '{}' with IndexError: {}\".format(func_name, e))\n    except Exception as e:\n        logging.error(\"Caught exception during formatting of app '{}': {}\".format(func_name, e))\n        raise e\n\n    logging.debug(\"Executable: %s\", executable)\n\n    # Updating stdout, stderr if values passed at call time.\n\n    def open_std_fd(fdname):\n        # fdname is 'stdout' or 'stderr'\n        stdfspec = kwargs.get(fdname)  # spec is str name or tuple (name, mode)\n        if stdfspec is None:\n            return None\n        elif isinstance(stdfspec, str):\n            fname = stdfspec\n            mode = 'a+'\n        elif isinstance(stdfspec, tuple):\n            if len(stdfspec) != 2:\n                raise pe.BadStdStreamFile(\"std descriptor %s has incorrect tuple length %s\" % (fdname, len(stdfspec)), TypeError('Bad Tuple Length'))\n            fname, mode = stdfspec\n        else:\n            raise pe.BadStdStreamFile(\"std descriptor %s has unexpected type %s\" % (fdname, str(type(stdfspec))), TypeError('Bad Tuple Type'))\n        try:\n            fd = open(fname, mode)\n        except Exception as e:\n            raise pe.BadStdStreamFile(fname, e)\n        return fd\n\n    std_out = open_std_fd('stdout')\n    std_err = open_std_fd('stderr')\n    timeout = kwargs.get('walltime')\n\n    returncode = None\n    try:\n        proc = subprocess.Popen(executable, stdout=std_out, stderr=std_err, shell=True, executable='/bin/bash')\n        proc.wait(timeout=timeout)\n        returncode = proc.returncode\n\n    except subprocess.TimeoutExpired:\n        # print(\"Timeout\")\n        raise pe.AppTimeout(\"[{}] App exceeded walltime: {}\".format(func_name, timeout))\n\n    except Exception as e:\n        # print(\"Caught exception: \", e)\n        raise pe.AppException(\"[{}] App caught exception: {}\".format(func_name, proc.returncode), e)\n\n    if returncode != 0:\n        raise pe.AppFailure(\"[{}] App failed with exit code: {}\".format(func_name, proc.returncode), proc.returncode)\n\n    # TODO : Add support for globs here\n\n    missing = []\n    for outputfile in kwargs.get('outputs', []):\n        fpath = outputfile\n        if type(outputfile) != str:\n            fpath = outputfile.filepath\n\n        if not os.path.exists(fpath):\n            missing.extend([outputfile])\n\n    if missing:\n        raise pe.MissingOutputs(\"[{}] Missing outputs\".format(func_name), missing)\n\n    # exec_duration = time.time() - start_t\n    return returncode"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef submit(self, command, blocksize, tasks_per_node, job_name=\"parsl.auto\"):\n        ''' The submit method takes the command string to be executed upon\n        instantiation of a resource most often to start a pilot.\n\n        Args :\n             - command (str) : The bash command string to be executed.\n             - blocksize (int) : Blocksize to be requested\n             - tasks_per_node (int) : command invocations to be launched per node\n\n        KWargs:\n             - job_name (str) : Human friendly name to be assigned to the job request\n\n        Returns:\n             - A job identifier, this could be an integer, string etc\n\n        Raises:\n             - ExecutionProviderException or its subclasses\n        '''\n        wrapped_cmd = self.launcher(command,\n                                    tasks_per_node,\n                                    1)\n\n        instance, name = self.create_instance(command=wrapped_cmd)\n        self.provisioned_blocks += 1\n        self.resources[name] = {\"job_id\": name, \"status\": translate_table[instance['status']]}\n        return name", "response": "The submit method creates a new instance of the appropriate class and assigns it to the resource list."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef status(self, job_ids):\n        ''' Get the status of a list of jobs identified by the job identifiers\n        returned from the submit request.\n\n        Args:\n             - job_ids (list) : A list of job identifiers\n\n        Returns:\n             - A list of status from ['PENDING', 'RUNNING', 'CANCELLED', 'COMPLETED',\n               'FAILED', 'TIMEOUT'] corresponding to each job_id in the job_ids list.\n\n        Raises:\n             - ExecutionProviderException or its subclasses\n\n        '''\n        statuses = []\n        for job_id in job_ids:\n            instance = self.client.instances().get(instance=job_id, project=self.project_id, zone=self.zone).execute()\n            self.resources[job_id]['status'] = translate_table[instance['status']]\n            statuses.append(translate_table[instance['status']])\n        return statuses", "response": "Get the status of a list of jobs identified by the job identifiers\n        Returns a list of status strings"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncanceling the resources identified by the job_ids provided by the user.", "response": "def cancel(self, job_ids):\n        ''' Cancels the resources identified by the job_ids provided by the user.\n\n        Args:\n             - job_ids (list): A list of job identifiers\n\n        Returns:\n             - A list of status from cancelling the job which can be True, False\n\n        Raises:\n             - ExecutionProviderException or its subclasses\n        '''\n        statuses = []\n        for job_id in job_ids:\n            try:\n                self.delete_instance(job_id)\n                statuses.append(True)\n                self.provisioned_blocks -= 1\n            except Exception:\n                statuses.append(False)\n        return statuses"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef _queue_management_worker(self):\n        while True:\n            logger.debug(\"[MTHREAD] Management thread active\")\n            try:\n                msg = self.incoming_q.get(block=True, timeout=1)\n\n            except queue.Empty:\n                # Timed out.\n                pass\n\n            except IOError as e:\n                logger.debug(\"[MTHREAD] Caught broken queue with exception code {}: {}\".format(e.errno, e))\n                return\n\n            except Exception as e:\n                logger.debug(\"[MTHREAD] Caught unknown exception: {}\".format(e))\n\n            else:\n\n                if msg is None:\n                    logger.debug(\"[MTHREAD] Got None\")\n                    return\n\n                else:\n                    logger.debug(\"[MTHREAD] Received message: {}\".format(msg))\n                    task_fut = self.tasks[msg['task_id']]\n                    if 'result' in msg:\n                        result, _ = deserialize_object(msg['result'])\n                        task_fut.set_result(result)\n\n                    elif 'exception' in msg:\n                        exception, _ = deserialize_object(msg['exception'])\n                        task_fut.set_exception(exception)\n\n            if not self.is_alive:\n                break", "response": "This function is the main thread that handles the management queue. It handles the messages and handles them."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef shutdown(self):\n        self.is_alive = False\n        logging.debug(\"Waking management thread\")\n        self.incoming_q.put(None)  # Wake up the thread\n        self._queue_management_thread.join()  # Force join\n        logging.debug(\"Exiting thread\")\n        self.worker.join()\n        return True", "response": "Shutdown method to kill the threads and workers."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nsubmit a function to the outgoing_q.", "response": "def submit(self, func, *args, **kwargs):\n        \"\"\"Submits work to the the outgoing_q.\n\n        The outgoing_q is an external process listens on this\n        queue for new work. This method is simply pass through and behaves like a\n        submit call as described here `Python docs: <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_\n\n        Args:\n            - func (callable) : Callable function\n            - *args (list) : List of arbitrary positional arguments.\n\n        Kwargs:\n            - **kwargs (dict) : A dictionary of arbitrary keyword args for func.\n\n        Returns:\n              Future\n        \"\"\"\n        task_id = uuid.uuid4()\n\n        logger.debug(\"Pushing function {} to queue with args {}\".format(func, args))\n\n        self.tasks[task_id] = Future()\n\n        fn_buf = pack_apply_message(func, args, kwargs,\n                                    buffer_threshold=1024 * 1024,\n                                    item_threshold=1024)\n\n        msg = {\"task_id\": task_id,\n               \"buffer\": fn_buf}\n\n        # Post task to the the outgoing queue\n        self.outgoing_q.put(msg)\n\n        # Return the future\n        return self.tasks[task_id]"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreturns the filepath on the side where it is called from.", "response": "def filepath(self):\n        \"\"\"Return the resolved filepath on the side where it is called from.\n\n        The appropriate filepath will be returned when called from within\n        an app running remotely as well as regular python on the client side.\n\n        Args:\n            - self\n        Returns:\n             - filepath (string)\n        \"\"\"\n        if hasattr(self, 'local_path'):\n            return self.local_path\n\n        if self.scheme in ['ftp', 'http', 'https', 'globus']:\n            return self.filename\n        elif self.scheme in ['file']:\n            return self.path\n        else:\n            raise Exception('Cannot return filepath for unknown scheme {}'.format(self.scheme))"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef execute_wait(self, cmd, walltime=None, envs={}):\n        ''' Synchronously execute a commandline string on the shell.\n\n        Args:\n            - cmd (string) : Commandline string to execute\n            - walltime (int) : walltime in seconds, this is not really used now.\n\n        Kwargs:\n            - envs (dict) : Dictionary of env variables. This will be used\n              to override the envs set at channel initialization.\n\n        Returns:\n            - retcode : Return code from the execution, -1 on fail\n            - stdout  : stdout string\n            - stderr  : stderr string\n\n        Raises:\n        None.\n        '''\n        retcode = -1\n        stdout = None\n        stderr = None\n\n        current_env = copy.deepcopy(self._envs)\n        current_env.update(envs)\n\n        try:\n            proc = subprocess.Popen(\n                cmd,\n                stdout=subprocess.PIPE,\n                stderr=subprocess.PIPE,\n                cwd=self.userhome,\n                env=current_env,\n                shell=True\n            )\n            proc.wait(timeout=walltime)\n            stdout = proc.stdout.read()\n            stderr = proc.stderr.read()\n            retcode = proc.returncode\n\n        except Exception as e:\n            print(\"Caught exception: {0}\".format(e))\n            logger.warn(\"Execution of command [%s] failed due to \\n %s \", cmd, e)\n            # Set retcode to non-zero so that this can be handled in the provider.\n            if retcode == 0:\n                retcode = -1\n            return (retcode, None, None)\n\n        return (retcode, stdout.decode(\"utf-8\"), stderr.decode(\"utf-8\"))", "response": "Executes a command on the shell and returns the output of the command."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef push_file(self, source, dest_dir):\n        ''' If the source files dirpath is the same as dest_dir, a copy\n        is not necessary, and nothing is done. Else a copy is made.\n\n        Args:\n            - source (string) : Path to the source file\n            - dest_dir (string) : Path to the directory to which the files is to be copied\n\n        Returns:\n            - destination_path (String) : Absolute path of the destination file\n\n        Raises:\n            - FileCopyException : If file copy failed.\n        '''\n\n        local_dest = dest_dir + '/' + os.path.basename(source)\n\n        # Only attempt to copy if the target dir and source dir are different\n        if os.path.dirname(source) != dest_dir:\n            try:\n                shutil.copyfile(source, local_dest)\n                os.chmod(local_dest, 0o777)\n\n            except OSError as e:\n                raise FileCopyException(e, self.hostname)\n\n        return local_dest", "response": "Pushes a file from source to dest_dir."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef make_rundir(path):\n    try:\n        if not os.path.exists(path):\n            os.makedirs(path)\n\n        prev_rundirs = glob(os.path.join(path, \"[0-9]*\"))\n\n        current_rundir = os.path.join(path, '000')\n\n        if prev_rundirs:\n            # Since we globbed on files named as 0-9\n            x = sorted([int(os.path.basename(x)) for x in prev_rundirs])[-1]\n            current_rundir = os.path.join(path, '{0:03}'.format(x + 1))\n\n        os.makedirs(current_rundir)\n        logger.debug(\"Parsl run initializing in rundir: {0}\".format(current_rundir))\n        return os.path.abspath(current_rundir)\n\n    except Exception as e:\n        logger.error(\"Failed to create a run directory\")\n        logger.error(\"Error: {0}\".format(e))\n        raise", "response": "Create a rundir for the run."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef put(self, task_id, buffer):\n        task_id_bytes = task_id.to_bytes(4, \"little\")\n        message = [b\"\", task_id_bytes] + buffer\n\n        self.zmq_socket.send_multipart(message)\n        logger.debug(\"Sent task {}\".format(task_id))", "response": "Put a new task into the queue."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef monitor(pid, task_id, monitoring_hub_url, run_id, sleep_dur=10):\n    import psutil\n\n    radio = UDPRadio(monitoring_hub_url,\n                     source_id=task_id)\n\n    # these values are simple to log. Other information is available in special formats such as memory below.\n    simple = [\"cpu_num\", 'cpu_percent', 'create_time', 'cwd', 'exe', 'memory_percent', 'nice', 'name', 'num_threads', 'pid', 'ppid', 'status', 'username']\n    # values that can be summed up to see total resources used by task process and its children\n    summable_values = ['cpu_percent', 'memory_percent', 'num_threads']\n\n    pm = psutil.Process(pid)\n    pm.cpu_percent()\n\n    first_msg = True\n\n    while True:\n        try:\n            d = {\"psutil_process_\" + str(k): v for k, v in pm.as_dict().items() if k in simple}\n            d[\"run_id\"] = run_id\n            d[\"task_id\"] = task_id\n            d['resource_monitoring_interval'] = sleep_dur\n            d['first_msg'] = first_msg\n            d['timestamp'] = datetime.datetime.now()\n            children = pm.children(recursive=True)\n            d[\"psutil_cpu_count\"] = psutil.cpu_count()\n            d['psutil_process_memory_virtual'] = pm.memory_info().vms\n            d['psutil_process_memory_resident'] = pm.memory_info().rss\n            d['psutil_process_time_user'] = pm.cpu_times().user\n            d['psutil_process_time_system'] = pm.cpu_times().system\n            d['psutil_process_children_count'] = len(children)\n            try:\n                d['psutil_process_disk_write'] = pm.io_counters().write_bytes\n                d['psutil_process_disk_read'] = pm.io_counters().read_bytes\n            except psutil._exceptions.AccessDenied:\n                # occassionally pid temp files that hold this information are unvailable to be read so set to zero\n                d['psutil_process_disk_write'] = 0\n                d['psutil_process_disk_read'] = 0\n            for child in children:\n                for k, v in child.as_dict(attrs=summable_values).items():\n                    d['psutil_process_' + str(k)] += v\n                d['psutil_process_time_user'] += child.cpu_times().user\n                d['psutil_process_time_system'] += child.cpu_times().system\n                d['psutil_process_memory_virtual'] += child.memory_info().vms\n                d['psutil_process_memory_resident'] += child.memory_info().rss\n                try:\n                    d['psutil_process_disk_write'] += child.io_counters().write_bytes\n                    d['psutil_process_disk_read'] += child.io_counters().read_bytes\n                except psutil._exceptions.AccessDenied:\n                    # occassionally pid temp files that hold this information are unvailable to be read so add zero\n                    d['psutil_process_disk_write'] += 0\n                    d['psutil_process_disk_read'] += 0\n\n        finally:\n            radio.send(MessageType.TASK_INFO, task_id, d)\n            time.sleep(sleep_dur)\n            first_msg = False", "response": "Internal helper function to monitor the task s resources by pointing psutil to the task s pid and watching its children."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nsend a message to the UDP receiver.", "response": "def send(self, message_type, task_id, message):\n        \"\"\" Sends a message to the UDP receiver\n\n        Parameter\n        ---------\n\n        message_type: monitoring.MessageType (enum)\n            In this case message type is RESOURCE_INFO most often\n        task_id: int\n            Task identifier of the task for which resource monitoring is being reported\n        message: object\n            Arbitrary pickle-able object that is to be sent\n\n        Returns:\n            # bytes sent\n        \"\"\"\n        x = 0\n        try:\n            buffer = pickle.dumps((self.source_id,   # Identifier for manager\n                                   int(time.time()),  # epoch timestamp\n                                   message_type,\n                                   message))\n        except Exception as e:\n            print(\"Exception during pickling {}\".format(e))\n            return\n\n        try:\n            x = self.sock.sendto(buffer, (self.ip, self.port))\n        except socket.timeout:\n            print(\"Could not send message within timeout limit\")\n            return False\n        return x"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nexecute a command on the shell and wait for the command to complete.", "response": "def execute_wait(self, cmd, walltime=2, envs={}):\n        ''' Synchronously execute a commandline string on the shell.\n\n        Args:\n            - cmd (string) : Commandline string to execute\n            - walltime (int) : walltime in seconds\n\n        Kwargs:\n            - envs (dict) : Dictionary of env variables\n\n        Returns:\n            - retcode : Return code from the execution, -1 on fail\n            - stdout  : stdout string\n            - stderr  : stderr string\n\n        Raises:\n        None.\n        '''\n\n        # Execute the command\n        stdin, stdout, stderr = self.ssh_client.exec_command(\n            self.prepend_envs(cmd, envs), bufsize=-1, timeout=walltime\n        )\n        # Block on exit status from the command\n        exit_status = stdout.channel.recv_exit_status()\n        return exit_status, stdout.read().decode(\"utf-8\"), stderr.read().decode(\"utf-8\")"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nexecute asynchronousely without waiting for exitcode", "response": "def execute_no_wait(self, cmd, walltime=2, envs={}):\n        ''' Execute asynchronousely without waiting for exitcode\n\n        Args:\n            - cmd (string): Commandline string to be executed on the remote side\n            - walltime (int): timeout to exec_command\n\n        KWargs:\n            - envs (dict): A dictionary of env variables\n\n        Returns:\n            - None, stdout (readable stream), stderr (readable stream)\n\n        Raises:\n            - ChannelExecFailed (reason)\n        '''\n\n        # Execute the command\n        stdin, stdout, stderr = self.ssh_client.exec_command(\n            self.prepend_envs(cmd, envs), bufsize=-1, timeout=walltime\n        )\n        return None, stdout, stderr"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef push_file(self, local_source, remote_dir):\n        ''' Transport a local file to a directory on a remote machine\n\n        Args:\n            - local_source (string): Path\n            - remote_dir (string): Remote path\n\n        Returns:\n            - str: Path to copied file on remote machine\n\n        Raises:\n            - BadScriptPath : if script path on the remote side is bad\n            - BadPermsScriptPath : You do not have perms to make the channel script dir\n            - FileCopyException : FileCopy failed.\n\n        '''\n        remote_dest = remote_dir + '/' + os.path.basename(local_source)\n\n        try:\n            self.makedirs(remote_dir, exist_ok=True)\n        except IOError as e:\n            logger.exception(\"Pushing {0} to {1} failed\".format(local_source, remote_dir))\n            if e.errno == 2:\n                raise BadScriptPath(e, self.hostname)\n            elif e.errno == 13:\n                raise BadPermsScriptPath(e, self.hostname)\n            else:\n                logger.exception(\"File push failed due to SFTP client failure\")\n                raise FileCopyException(e, self.hostname)\n        try:\n            self.sftp_client.put(local_source, remote_dest, confirm=True)\n            # Set perm because some systems require the script to be executable\n            self.sftp_client.chmod(remote_dest, 0o777)\n        except Exception as e:\n            logger.exception(\"File push from local source {} to remote destination {} failed\".format(\n                local_source, remote_dest))\n            raise FileCopyException(e, self.hostname)\n\n        return remote_dest", "response": "Transport a local file to a directory on a remote machine"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef pull_file(self, remote_source, local_dir):\n        ''' Transport file on the remote side to a local directory\n\n        Args:\n            - remote_source (string): remote_source\n            - local_dir (string): Local directory to copy to\n\n\n        Returns:\n            - str: Local path to file\n\n        Raises:\n            - FileExists : Name collision at local directory.\n            - FileCopyException : FileCopy failed.\n        '''\n\n        local_dest = local_dir + '/' + os.path.basename(remote_source)\n\n        try:\n            os.makedirs(local_dir)\n        except OSError as e:\n            if e.errno != errno.EEXIST:\n                logger.exception(\"Failed to create script_dir: {0}\".format(script_dir))\n                raise BadScriptPath(e, self.hostname)\n\n        # Easier to check this than to waste time trying to pull file and\n        # realize there's a problem.\n        if os.path.exists(local_dest):\n            logger.exception(\"Remote file copy will overwrite a local file:{0}\".format(local_dest))\n            raise FileExists(None, self.hostname, filename=local_dest)\n\n        try:\n            self.sftp_client.get(remote_source, local_dest)\n        except Exception as e:\n            logger.exception(\"File pull failed\")\n            raise FileCopyException(e, self.hostname)\n\n        return local_dest", "response": "Pulls a file from the remote side to a local directory."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef isdir(self, path):\n        result = True\n        try:\n            self.sftp_client.lstat(path)\n        except FileNotFoundError:\n            result = False\n\n        return result", "response": "Return True if the path refers to an existing directory."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef makedirs(self, path, mode=511, exist_ok=False):\n        if exist_ok is False and self.isdir(path):\n            raise OSError('Target directory {} already exists'.format(path))\n\n        self.execute_wait('mkdir -p {}'.format(path))\n        self.sftp_client.chmod(path, mode)", "response": "Create a directory on the remote side."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef _wake_up_timer(self, kill_event):\n\n        while True:\n            prev = self._wake_up_time\n\n            # Waiting for the event returns True only when the event\n            # is set, usually by the parent thread\n            time_to_die = kill_event.wait(float(max(prev - time.time(), 0)))\n\n            if time_to_die:\n                return\n\n            if prev == self._wake_up_time:\n                self.make_callback(kind='timer')\n            else:\n                print(\"Sleeping a bit more\")", "response": "Internal function that waits for an event to be set and then waits for the event to be set."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nnotifying FlowControl system that there is an event.", "response": "def notify(self, event_id):\n        \"\"\"Let the FlowControl system know that there is an event.\"\"\"\n        self._event_buffer.extend([event_id])\n        self._event_count += 1\n        if self._event_count >= self.threshold:\n            logger.debug(\"Eventcount >= threshold\")\n            self.make_callback(kind=\"event\")"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nmakes the callback and resets the timer.", "response": "def make_callback(self, kind=None):\n        \"\"\"Makes the callback and resets the timer.\n\n        KWargs:\n               - kind (str): Default=None, used to pass information on what\n                 triggered the callback\n        \"\"\"\n        self._wake_up_time = time.time() + self.interval\n        self.callback(tasks=self._event_buffer, kind=kind)\n        self._event_buffer = []"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nmake the callback and resets the timer.", "response": "def make_callback(self, kind=None):\n        \"\"\"Makes the callback and resets the timer.\n        \"\"\"\n        self._wake_up_time = time.time() + self.interval\n        self.callback(*self.cb_args)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nreturning the IP address of the given interface name e. g. eth0", "response": "def address_by_interface(ifname):\n    \"\"\"Returns the IP address of the given interface name, e.g. 'eth0'\n\n    Parameters\n    ----------\n    ifname : str\n        Name of the interface whose address is to be returned. Required.\n\n    Taken from this Stack Overflow answer: https://stackoverflow.com/questions/24196932/how-can-i-get-the-ip-address-of-eth0-in-python#24196955\n    \"\"\"\n    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n    return socket.inet_ntoa(fcntl.ioctl(\n        s.fileno(),\n        0x8915,  # SIOCGIFADDR\n        struct.pack('256s', bytes(ifname[:15], 'utf-8'))\n    )[20:24])"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nsubmits a job to the cluster", "response": "def submit(self, cmd_string, blocksize, tasks_per_node, job_name=\"parsl\"):\n        \"\"\" Submit a job\n        Args:\n             - cmd_string  :(String) - Name of the container to initiate\n             - blocksize   :(float) - Number of replicas\n             - tasks_per_node (int) : command invocations to be launched per node\n\n        Kwargs:\n             - job_name (String): Name for job, must be unique\n        Returns:\n             - None: At capacity, cannot provision more\n             - job_id: (string) Identifier for the job\n        \"\"\"\n        if not self.resources:\n            cur_timestamp = str(time.time() * 1000).split(\".\")[0]\n            job_name = \"{0}-{1}\".format(job_name, cur_timestamp)\n\n            if not self.deployment_name:\n                deployment_name = '{}-deployment'.format(job_name)\n            else:\n                deployment_name = '{}-{}-deployment'.format(self.deployment_name,\n                                                            cur_timestamp)\n\n            formatted_cmd = template_string.format(command=cmd_string,\n                                                   worker_init=self.worker_init)\n\n            self.deployment_obj = self._create_deployment_object(job_name,\n                                                                 self.image,\n                                                                 deployment_name,\n                                                                 cmd_string=formatted_cmd,\n                                                                 replicas=self.init_blocks,\n                                                                 volumes=self.persistent_volumes)\n            logger.debug(\"Deployment name :{}\".format(deployment_name))\n            self._create_deployment(self.deployment_obj)\n            self.resources[deployment_name] = {'status': 'RUNNING',\n                                               'pods': self.init_blocks}\n\n        return deployment_name"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating a kubernetes deployment object for the job.", "response": "def _create_deployment_object(self, job_name, job_image,\n                                  deployment_name, port=80,\n                                  replicas=1,\n                                  cmd_string=None,\n                                  engine_json_file='~/.ipython/profile_default/security/ipcontroller-engine.json',\n                                  engine_dir='.',\n                                  volumes=[]):\n        \"\"\" Create a kubernetes deployment for the job.\n        Args:\n              - job_name (string) : Name of the job and deployment\n              - job_image (string) : Docker image to launch\n        KWargs:\n             - port (integer) : Container port\n             - replicas : Number of replica containers to maintain\n        Returns:\n              - True: The deployment object to launch\n        \"\"\"\n\n        # sorry, quick hack that doesn't pass this stuff through to test it works.\n        # TODO it also doesn't only add what is set :(\n        security_context = None\n        if self.user_id and self.group_id:\n            security_context = client.V1SecurityContext(run_as_group=self.group_id,\n                                                        run_as_user=self.user_id,\n                                                        run_as_non_root=self.run_as_non_root)\n\n        # Create the enviornment variables and command to initiate IPP\n        environment_vars = client.V1EnvVar(name=\"TEST\", value=\"SOME DATA\")\n\n        launch_args = [\"-c\", \"{0}; /app/deploy.sh;\".format(cmd_string)]\n\n        volume_mounts = []\n        # Create mount paths for the volumes\n        for volume in volumes:\n            volume_mounts.append(client.V1VolumeMount(mount_path=volume[1],\n                                                      name=volume[0]))\n        # Configureate Pod template container\n        container = None\n        if security_context:\n            container = client.V1Container(\n                name=job_name,\n                image=job_image,\n                ports=[client.V1ContainerPort(container_port=port)],\n                volume_mounts=volume_mounts,\n                command=['/bin/bash'],\n                args=launch_args,\n                env=[environment_vars],\n                security_context=security_context)\n        else:\n            container = client.V1Container(\n                name=job_name,\n                image=job_image,\n                ports=[client.V1ContainerPort(container_port=port)],\n                volume_mounts=volume_mounts,\n                command=['/bin/bash'],\n                args=launch_args,\n                env=[environment_vars])\n        # Create a secret to enable pulling images from secure repositories\n        secret = None\n        if self.secret:\n            secret = client.V1LocalObjectReference(name=self.secret)\n\n        # Create list of volumes from (pvc, mount) tuples\n        volume_defs = []\n        for volume in volumes:\n            volume_defs.append(client.V1Volume(name=volume[0],\n                                               persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(\n                                                   claim_name=volume[0])))\n\n        # Create and configurate a spec section\n        template = client.V1PodTemplateSpec(\n            metadata=client.V1ObjectMeta(labels={\"app\": job_name}),\n            spec=client.V1PodSpec(containers=[container],\n                                  image_pull_secrets=[secret],\n                                  volumes=volume_defs\n                                  ))\n\n        # Create the specification of deployment\n        spec = client.ExtensionsV1beta1DeploymentSpec(replicas=replicas,\n                                                      template=template)\n\n        # Instantiate the deployment object\n        deployment = client.ExtensionsV1beta1Deployment(\n            api_version=\"extensions/v1beta1\",\n            kind=\"Deployment\",\n            metadata=client.V1ObjectMeta(name=deployment_name),\n            spec=spec)\n\n        return deployment"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncreating the kubernetes deployment", "response": "def _create_deployment(self, deployment):\n        \"\"\" Create the kubernetes deployment \"\"\"\n\n        api_response = self.kube_client.create_namespaced_deployment(\n            body=deployment,\n            namespace=self.namespace)\n\n        logger.debug(\"Deployment created. status='{0}'\".format(str(api_response.status)))"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ndelete the specified deployment.", "response": "def _delete_deployment(self, deployment_name):\n        \"\"\" Delete deployment \"\"\"\n\n        api_response = self.kube_client.delete_namespaced_deployment(\n            name=deployment_name,\n            namespace=self.namespace,\n            body=client.V1DeleteOptions(\n                propagation_policy='Foreground',\n                grace_period_seconds=5))\n\n        logger.debug(\"Deployment deleted. status='{0}'\".format(\n            str(api_response.status)))"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef initialize_scaling(self):\n        debug_opts = \"--debug\" if self.worker_debug else \"\"\n        max_workers = \"\" if self.max_workers == float('inf') else \"--max_workers={}\".format(self.max_workers)\n\n        worker_logdir = \"{}/{}\".format(self.run_dir, self.label)\n        if self.worker_logdir_root is not None:\n            worker_logdir = \"{}/{}\".format(self.worker_logdir_root, self.label)\n\n        l_cmd = self.launch_cmd.format(debug=debug_opts,\n                                       prefetch_capacity=self.prefetch_capacity,\n                                       task_url=self.worker_task_url,\n                                       result_url=self.worker_result_url,\n                                       cores_per_worker=self.cores_per_worker,\n                                       max_workers=max_workers,\n                                       nodes_per_block=self.provider.nodes_per_block,\n                                       heartbeat_period=self.heartbeat_period,\n                                       heartbeat_threshold=self.heartbeat_threshold,\n                                       poll_period=self.poll_period,\n                                       logdir=worker_logdir)\n        self.launch_cmd = l_cmd\n        logger.debug(\"Launch command: {}\".format(self.launch_cmd))\n\n        self._scaling_enabled = self.provider.scaling_enabled\n        logger.debug(\"Starting HighThroughputExecutor with provider:\\n%s\", self.provider)\n        if hasattr(self.provider, 'init_blocks'):\n            try:\n                self.scale_out(blocks=self.provider.init_blocks)\n            except Exception as e:\n                logger.error(\"Scaling out failed: {}\".format(e))\n                raise e", "response": "Initialize the launch command and call the scale_out method."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nstarting the Interchange process and connect to it.", "response": "def start(self):\n        \"\"\"Create the Interchange process and connect to it.\n        \"\"\"\n        self.outgoing_q = zmq_pipes.TasksOutgoing(\"127.0.0.1\", self.interchange_port_range)\n        self.incoming_q = zmq_pipes.ResultsIncoming(\"127.0.0.1\", self.interchange_port_range)\n        self.command_client = zmq_pipes.CommandClient(\"127.0.0.1\", self.interchange_port_range)\n\n        self.is_alive = True\n\n        self._executor_bad_state = threading.Event()\n        self._executor_exception = None\n        self._queue_management_thread = None\n        self._start_queue_management_thread()\n        self._start_local_queue_process()\n\n        logger.debug(\"Created management thread: {}\".format(self._queue_management_thread))\n\n        if self.provider:\n            self.initialize_scaling()\n        else:\n            self._scaling_enabled = False\n            logger.debug(\"Starting HighThroughputExecutor with no provider\")"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _queue_management_worker(self):\n        logger.debug(\"[MTHREAD] queue management worker starting\")\n\n        while not self._executor_bad_state.is_set():\n            try:\n                msgs = self.incoming_q.get(timeout=1)\n                # logger.debug(\"[MTHREAD] get has returned {}\".format(len(msgs)))\n\n            except queue.Empty:\n                logger.debug(\"[MTHREAD] queue empty\")\n                # Timed out.\n                pass\n\n            except IOError as e:\n                logger.exception(\"[MTHREAD] Caught broken queue with exception code {}: {}\".format(e.errno, e))\n                return\n\n            except Exception as e:\n                logger.exception(\"[MTHREAD] Caught unknown exception: {}\".format(e))\n                return\n\n            else:\n\n                if msgs is None:\n                    logger.debug(\"[MTHREAD] Got None, exiting\")\n                    return\n\n                else:\n                    for serialized_msg in msgs:\n                        try:\n                            msg = pickle.loads(serialized_msg)\n                            tid = msg['task_id']\n                        except pickle.UnpicklingError:\n                            raise BadMessage(\"Message received could not be unpickled\")\n\n                        except Exception:\n                            raise BadMessage(\"Message received does not contain 'task_id' field\")\n\n                        if tid == -1 and 'exception' in msg:\n                            logger.warning(\"Executor shutting down due to exception from interchange\")\n                            self._executor_exception, _ = deserialize_object(msg['exception'])\n                            logger.exception(\"Exception: {}\".format(self._executor_exception))\n                            # Set bad state to prevent new tasks from being submitted\n                            self._executor_bad_state.set()\n                            # We set all current tasks to this exception to make sure that\n                            # this is raised in the main context.\n                            for task in self.tasks:\n                                self.tasks[task].set_exception(self._executor_exception)\n                            break\n\n                        task_fut = self.tasks[tid]\n\n                        if 'result' in msg:\n                            result, _ = deserialize_object(msg['result'])\n                            task_fut.set_result(result)\n\n                        elif 'exception' in msg:\n                            try:\n                                s, _ = deserialize_object(msg['exception'])\n                                # s should be a RemoteExceptionWrapper... so we can reraise it\n                                try:\n                                    s.reraise()\n                                except Exception as e:\n                                    task_fut.set_exception(e)\n                            except Exception as e:\n                                # TODO could be a proper wrapped exception?\n                                task_fut.set_exception(\n                                    DeserializationError(\"Received exception, but handling also threw an exception: {}\".format(e)))\n                        else:\n                            raise BadMessage(\"Message received is neither result or exception\")\n\n            if not self.is_alive:\n                break\n        logger.info(\"[MTHREAD] queue management worker finished\")", "response": "This function is called by the thread that handles the incoming queue for tasks and handles them."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nstarting the interchange process locally and uses an internal command queue to get the worker task and result ports that the interchange has bound to.", "response": "def _start_local_queue_process(self):\n        \"\"\" Starts the interchange process locally\n\n        Starts the interchange process locally and uses an internal command queue to\n        get the worker task and result ports that the interchange has bound to.\n        \"\"\"\n        comm_q = Queue(maxsize=10)\n        self.queue_proc = Process(target=interchange.starter,\n                                  args=(comm_q,),\n                                  kwargs={\"client_ports\": (self.outgoing_q.port,\n                                                           self.incoming_q.port,\n                                                           self.command_client.port),\n                                          \"worker_ports\": self.worker_ports,\n                                          \"worker_port_range\": self.worker_port_range,\n                                          \"logdir\": \"{}/{}\".format(self.run_dir, self.label),\n                                          \"suppress_failure\": self.suppress_failure,\n                                          \"heartbeat_threshold\": self.heartbeat_threshold,\n                                          \"poll_period\": self.poll_period,\n                                          \"logging_level\": logging.DEBUG if self.worker_debug else logging.INFO\n                                  },\n        )\n        self.queue_proc.start()\n        try:\n            (worker_task_port, worker_result_port) = comm_q.get(block=True, timeout=120)\n        except queue.Empty:\n            logger.error(\"Interchange has not completed initialization in 120s. Aborting\")\n            raise Exception(\"Interchange failed to start\")\n\n        self.worker_task_url = \"tcp://{}:{}\".format(self.address, worker_task_port)\n        self.worker_result_url = \"tcp://{}:{}\".format(self.address, worker_result_port)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nput a worker on hold.", "response": "def hold_worker(self, worker_id):\n        \"\"\"Puts a worker on hold, preventing scheduling of additional tasks to it.\n\n        This is called \"hold\" mostly because this only stops scheduling of tasks,\n        and does not actually kill the worker.\n\n        Parameters\n        ----------\n\n        worker_id : str\n            Worker id to be put on hold\n        \"\"\"\n        c = self.command_client.run(\"HOLD_WORKER;{}\".format(worker_id))\n        logger.debug(\"Sent hold request to worker: {}\".format(worker_id))\n        return c"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef _hold_block(self, block_id):\n\n        managers = self.connected_managers\n\n        for manager in managers:\n            if manager['block_id'] == block_id:\n                logger.debug(\"[HOLD_BLOCK]: Sending hold to manager:{}\".format(manager['manager']))\n                self.hold_worker(manager['manager'])", "response": "Sends a hold command to all managers which are in a specific block_id"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nscale out the number of blocks by blocks Raises NotImplementedError", "response": "def scale_out(self, blocks=1):\n        \"\"\"Scales out the number of blocks by \"blocks\"\n\n        Raises:\n             NotImplementedError\n        \"\"\"\n        r = []\n        for i in range(blocks):\n            if self.provider:\n                external_block_id = str(len(self.blocks))\n                launch_cmd = self.launch_cmd.format(block_id=external_block_id)\n                internal_block = self.provider.submit(launch_cmd, 1, 1)\n                logger.debug(\"Launched block {}->{}\".format(external_block_id, internal_block))\n                if not internal_block:\n                    raise(ScalingFailed(self.provider.label,\n                                        \"Attempts to provision nodes via provider has failed\"))\n                r.extend([external_block_id])\n                self.blocks[external_block_id] = internal_block\n            else:\n                logger.error(\"No execution provider available\")\n                r = None\n        return r"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef scale_in(self, blocks=None, block_ids=[]):\n\n        if block_ids:\n            block_ids_to_kill = block_ids\n        else:\n            block_ids_to_kill = list(self.blocks.keys())[:blocks]\n\n        # Hold the block\n        for block_id in block_ids_to_kill:\n            self._hold_block(block_id)\n\n        # Now kill via provider\n        to_kill = [self.blocks.pop(bid) for bid in block_ids_to_kill]\n\n        if self.provider:\n            r = self.provider.cancel(to_kill)\n\n        return r", "response": "This method scales in the number of active blocks by specified amount."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nreturns status of all blocks.", "response": "def status(self):\n        \"\"\"Return status of all blocks.\"\"\"\n\n        status = []\n        if self.provider:\n            status = self.provider.status(self.blocks.values())\n\n        return status"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef shutdown(self, hub=True, targets='all', block=False):\n\n        logger.info(\"Attempting HighThroughputExecutor shutdown\")\n        # self.outgoing_q.close()\n        # self.incoming_q.close()\n        self.queue_proc.terminate()\n        logger.info(\"Finished HighThroughputExecutor shutdown attempt\")\n        return True", "response": "This method is used to shutdown the executor."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nreading from the device into a buffer.", "response": "def readinto(self, buf, **kwargs):\n        \"\"\"\n        Read into ``buf`` from the device. The number of bytes read will be the\n        length of ``buf``.\n\n        If ``start`` or ``end`` is provided, then the buffer will be sliced\n        as if ``buf[start:end]``. This will not cause an allocation like\n        ``buf[start:end]`` will so it saves memory.\n\n        :param bytearray buffer: buffer to write into\n        :param int start: Index to start writing at\n        :param int end: Index to write up to but not include\n        \"\"\"\n        self.i2c.readfrom_into(self.device_address, buf, **kwargs)\n        if self._debug:\n            print(\"i2c_device.readinto:\", [hex(i) for i in buf])"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef write(self, buf, **kwargs):\n        self.i2c.writeto(self.device_address, buf, **kwargs)\n        if self._debug:\n            print(\"i2c_device.write:\", [hex(i) for i in buf])", "response": "Write the bytes from buffer to the device."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nwriting the bytes from out_buffer then immediately reads into in_buffer.", "response": "def write_then_readinto(self, out_buffer, in_buffer, *,\n                            out_start=0, out_end=None, in_start=0, in_end=None, stop=True):\n        \"\"\"\n        Write the bytes from ``out_buffer`` to the device, then immediately\n        reads into ``in_buffer`` from the device. The number of bytes read\n        will be the length of ``in_buffer``.\n        Transmits a stop bit after the write, if ``stop`` is set.\n\n        If ``out_start`` or ``out_end`` is provided, then the output buffer\n        will be sliced as if ``out_buffer[out_start:out_end]``. This will\n        not cause an allocation like ``buffer[out_start:out_end]`` will so\n        it saves memory.\n\n        If ``in_start`` or ``in_end`` is provided, then the input buffer\n        will be sliced as if ``in_buffer[in_start:in_end]``. This will not\n        cause an allocation like ``in_buffer[in_start:in_end]`` will so\n        it saves memory.\n\n        :param bytearray out_buffer: buffer containing the bytes to write\n        :param bytearray in_buffer: buffer containing the bytes to read into\n        :param int out_start: Index to start writing from\n        :param int out_end: Index to read up to but not include\n        :param int in_start: Index to start writing at\n        :param int in_end: Index to write up to but not include\n        :param bool stop: If true, output an I2C stop condition after the buffer is written\n        \"\"\"\n        if out_end is None:\n            out_end = len(out_buffer)\n        if in_end is None:\n            in_end = len(in_buffer)\n        if hasattr(self.i2c, 'writeto_then_readfrom'):\n            if self._debug:\n                print(\"i2c_device.writeto_then_readfrom.out_buffer:\",\n                      [hex(i) for i in out_buffer[out_start:out_end]])\n            # In linux, at least, this is a special kernel function call\n            self.i2c.writeto_then_readfrom(self.device_address, out_buffer, in_buffer,\n                                           out_start=out_start, out_end=out_end,\n                                           in_start=in_start, in_end=in_end, stop=stop)\n            if self._debug:\n                print(\"i2c_device.writeto_then_readfrom.in_buffer:\",\n                      [hex(i) for i in in_buffer[in_start:in_end]])\n        else:\n            # If we don't have a special implementation, we can fake it with two calls\n            self.write(out_buffer, start=out_start, end=out_end, stop=stop)\n            if self._debug:\n                print(\"i2c_device.write_then_readinto.write.out_buffer:\",\n                      [hex(i) for i in out_buffer[out_start:out_end]])\n            self.readinto(in_buffer, start=in_start, end=in_end)\n            if self._debug:\n                print(\"i2c_device.write_then_readinto.readinto.in_buffer:\",\n                      [hex(i) for i in in_buffer[in_start:in_end]])"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef read_sample(filename):\r\n    \r\n    file = open(filename, 'r')\r\n\r\n    sample = [[float(val) for val in line.split()] for line in file if len(line.strip()) > 0]\r\n    \r\n    file.close()\r\n    return sample", "response": "Read sample from a simple text file."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef calculate_distance_matrix(sample):\r\n\r\n    amount_rows = len(sample);\r\n    return [ [ euclidean_distance(sample[i], sample[j]) for j in range(amount_rows) ] for i in range(amount_rows) ];", "response": "Calculates the distance matrix for a sequence of data points."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nreads image as N - dimension matrix", "response": "def read_image(filename):\r\n    \"\"\"!\r\n    @brief Returns image as N-dimension (depends on the input image) matrix, where one element of list describes pixel.\r\n    \r\n    @param[in] filename (string): Path to image.\r\n    \r\n    @return (list) Pixels where each pixel described by list of RGB-values.\r\n    \r\n    \"\"\"\r\n    \r\n    with Image.open(filename) as image_source:\r\n        data = [list(pixel) for pixel in image_source.getdata()]\r\n        return data"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef rgb2gray(image_rgb_array):\r\n    \r\n    image_gray_array = [0.0] * len(image_rgb_array);\r\n    for index in range(0, len(image_rgb_array), 1):\r\n        image_gray_array[index] = float(image_rgb_array[index][0]) * 0.2989 + float(image_rgb_array[index][1]) * 0.5870 + float(image_rgb_array[index][2]) * 0.1140;\r\n    \r\n    return image_gray_array;", "response": "Returns image as 1 - dimension gray colored matrix where one element of list describes pixel."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef stretch_pattern(image_source):\r\n    wsize, hsize = image_source.size;\r\n    \r\n    # Crop digit exactly\r\n    (ws, hs, we, he) = gray_pattern_borders(image_source);\r\n    image_source = image_source.crop((ws, hs, we, he));\r\n    \r\n    # Stretch it to initial sizes\r\n    image_source = image_source.resize((wsize, hsize), Image.ANTIALIAS);\r\n    \r\n    # Transform image to simple array\r\n    data = [pixel for pixel in image_source.getdata()];\r\n    image_pattern = rgb2gray(data);\r\n    \r\n    return (image_pattern, image_source);", "response": "Returns stretched image as 1 - dimension gray colored matrix with size of input image."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef gray_pattern_borders(image):\r\n    \r\n    width, height = image.size;\r\n    \r\n    width_start = width;\r\n    width_end = 0;\r\n    height_start = height;\r\n    height_end = 0;\r\n    \r\n    row, col = 0, 0;\r\n    for pixel in image.getdata():\r\n        value = float(pixel[0]) * 0.2989 + float(pixel[1]) * 0.5870 + float(pixel[2]) * 0.1140;\r\n        \r\n        if (value < 128):\r\n            if (width_end < col): \r\n                width_end = col;\r\n            \r\n            if (height_end < row):\r\n                height_end = row;\r\n        \r\n            if (width_start > col):\r\n                width_start = col;\r\n            \r\n            if (height_start > row):\r\n                height_start = row;\r\n        \r\n        col += 1;\r\n        if (col >= width):\r\n            col = 0;\r\n            row += 1;\r\n\r\n    return (width_start, height_start, width_end + 1, height_end + 1);", "response": "Returns the coordinates of the gray pattern content on the input image."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ncalculate the average distance between nearest neighbors of a list of points.", "response": "def average_neighbor_distance(points, num_neigh):\r\n    \"\"\"!\r\n    @brief Returns average distance for establish links between specified number of nearest neighbors.\r\n    \r\n    @param[in] points (list): Input data, list of points where each point represented by list.\r\n    @param[in] num_neigh (uint): Number of neighbors that should be used for distance calculation.\r\n    \r\n    @return (double) Average distance for establish links between 'num_neigh' in data set 'points'.\r\n    \r\n    \"\"\"\r\n    \r\n    if num_neigh > len(points) - 1:\r\n        raise NameError('Impossible to calculate average distance to neighbors when number of object is less than number of neighbors.');\r\n    \r\n    dist_matrix = [ [ 0.0 for i in range(len(points)) ] for j in range(len(points)) ];\r\n    for i in range(0, len(points), 1):\r\n        for j in range(i + 1, len(points), 1):\r\n            distance = euclidean_distance(points[i], points[j]);\r\n            dist_matrix[i][j] = distance;\r\n            dist_matrix[j][i] = distance;\r\n            \r\n        dist_matrix[i] = sorted(dist_matrix[i]);\r\n\r\n    total_distance = 0;\r\n    for i in range(0, len(points), 1):\r\n        # start from 0 - first element is distance to itself.\r\n        for j in range(0, num_neigh, 1):\r\n            total_distance += dist_matrix[i][j + 1];\r\n            \r\n    return ( total_distance / (num_neigh * len(points)) );"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating the median of the input points using Euclidean distance.", "response": "def medoid(data, indexes=None, **kwargs):\r\n    \"\"\"!\r\n    @brief Calculate medoid for input points using Euclidean distance.\r\n    \r\n    @param[in] data (list): Set of points for that median should be calculated.\r\n    @param[in] indexes (list): Indexes of input set of points that will be taken into account during median calculation.\r\n    @param[in] **kwargs: Arbitrary keyword arguments (available arguments: 'metric', 'data_type').\r\n\r\n    <b>Keyword Args:</b><br>\r\n        - metric (distance_metric): Metric that is used for distance calculation between two points.\r\n        - data_type (string): Data type of input sample 'data' (available values: 'points', 'distance_matrix').\r\n\r\n    @return (uint) index of point in input set that corresponds to median.\r\n    \r\n    \"\"\"\r\n    \r\n    index_median = None\r\n    distance = float('Inf')\r\n\r\n    metric = kwargs.get('metric', type_metric.EUCLIDEAN_SQUARE)\r\n    data_type = kwargs.get('data_type', 'points')\r\n\r\n    if data_type == 'points':\r\n        calculator = lambda index1, index2: metric(data[index1], data[index2])\r\n    elif data_type == 'distance_matrix':\r\n        if isinstance(data, numpy.matrix):\r\n            calculator = lambda index1, index2: data.item(index1, index2)\r\n\r\n        else:\r\n            calculator = lambda index1, index2: data[index1][index2]\r\n    else:\r\n        raise TypeError(\"Unknown type of data is specified '%s'.\" % data_type)\r\n\r\n    if indexes is None:\r\n        range_points = range(len(data))\r\n    else:\r\n        range_points = indexes\r\n    \r\n    for index_candidate in range_points:\r\n        distance_candidate = 0.0\r\n        for index in range_points:\r\n            distance_candidate += calculator(index_candidate, index)\r\n        \r\n        if distance_candidate < distance:\r\n            distance = distance_candidate\r\n            index_median = index_candidate\r\n    \r\n    return index_median"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef euclidean_distance_square(a, b):\r\n    \r\n    if ( ((type(a) == float) and (type(b) == float)) or ((type(a) == int) and (type(b) == int)) ):\r\n        return (a - b)**2.0;\r\n    \r\n    distance = 0.0;\r\n    for i in range(0, len(a)):\r\n        distance += (a[i] - b[i])**2.0;\r\n        \r\n    return distance;", "response": "Calculates the Euclidian distance between two vectors a and b."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef manhattan_distance(a, b):\r\n    \r\n    if ( ((type(a) == float) and (type(b) == float)) or ((type(a) == int) and (type(b) == int)) ):\r\n        return abs(a - b);\r\n    \r\n    distance = 0.0;\r\n    dimension = len(a);\r\n    \r\n    for i in range(0, dimension):\r\n        distance += abs(a[i] - b[i]);\r\n    \r\n    return distance;", "response": "Calculates the Manhattan distance between two vectors."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef average_inter_cluster_distance(cluster1, cluster2, data = None):\r\n    \r\n    distance = 0.0;\r\n    \r\n    if (data is None):\r\n        for i in range(len(cluster1)):\r\n            for j in range(len(cluster2)):\r\n                distance += euclidean_distance_square(cluster1[i], cluster2[j]);\r\n    else:\r\n        for i in range(len(cluster1)):\r\n            for j in range(len(cluster2)):\r\n                distance += euclidean_distance_square(data[ cluster1[i]], data[ cluster2[j]]);\r\n    \r\n    distance /= float(len(cluster1) * len(cluster2));\r\n    return distance ** 0.5;", "response": "Calculates the average inter - cluster distance between two clusters."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncalculate the average intra - cluster distance between two clusters.", "response": "def average_intra_cluster_distance(cluster1, cluster2, data=None):\r\n    \"\"\"!\r\n    @brief Calculates average intra-cluster distance between two clusters.\r\n    @details Clusters can be represented by list of coordinates (in this case data shouldn't be specified),\r\n             or by list of indexes of points from the data (represented by list of points), in this case \r\n             data should be specified.\r\n    \r\n    @param[in] cluster1 (list): The first cluster.\r\n    @param[in] cluster2 (list): The second cluster.\r\n    @param[in] data (list): If specified than elements of clusters will be used as indexes,\r\n               otherwise elements of cluster will be considered as points.\r\n    \r\n    @return (double) Average intra-cluster distance between two clusters.\r\n    \r\n    \"\"\"\r\n        \r\n    distance = 0.0\r\n    \r\n    for i in range(len(cluster1) + len(cluster2)):\r\n        for j in range(len(cluster1) + len(cluster2)):\r\n            if data is None:\r\n                # the first point\r\n                if i < len(cluster1):\r\n                    first_point = cluster1[i]\r\n                else:\r\n                    first_point = cluster2[i - len(cluster1)]\r\n                \r\n                # the second point\r\n                if j < len(cluster1):\r\n                    second_point = cluster1[j]\r\n                else:\r\n                    second_point = cluster2[j - len(cluster1)]\r\n                \r\n            else:\r\n                # the first point\r\n                if i < len(cluster1):\r\n                    first_point = data[cluster1[i]]\r\n                else:\r\n                    first_point = data[cluster2[i - len(cluster1)]]\r\n            \r\n                if j < len(cluster1):\r\n                    second_point = data[cluster1[j]]\r\n                else:\r\n                    second_point = data[cluster2[j - len(cluster1)]]\r\n            \r\n            distance += euclidean_distance_square(first_point, second_point)\r\n    \r\n    distance /= float((len(cluster1) + len(cluster2)) * (len(cluster1) + len(cluster2) - 1.0))\r\n    return distance ** 0.5"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\ncalculating the variance increase distance between two clusters.", "response": "def variance_increase_distance(cluster1, cluster2, data = None):\r\n    \"\"\"!\r\n    @brief Calculates variance increase distance between two clusters.\r\n    @details Clusters can be represented by list of coordinates (in this case data shouldn't be specified),\r\n             or by list of indexes of points from the data (represented by list of points), in this case \r\n             data should be specified.\r\n    \r\n    @param[in] cluster1 (list): The first cluster.\r\n    @param[in] cluster2 (list): The second cluster.\r\n    @param[in] data (list): If specified than elements of clusters will be used as indexes,\r\n               otherwise elements of cluster will be considered as points.\r\n    \r\n    @return (double) Average variance increase distance between two clusters.\r\n    \r\n    \"\"\"\r\n    \r\n    # calculate local sum\r\n    if data is None:\r\n        member_cluster1 = [0.0] * len(cluster1[0])\r\n        member_cluster2 = [0.0] * len(cluster2[0])\r\n        \r\n    else:\r\n        member_cluster1 = [0.0] * len(data[0])\r\n        member_cluster2 = [0.0] * len(data[0])\r\n    \r\n    for i in range(len(cluster1)):\r\n        if data is None:\r\n            member_cluster1 = list_math_addition(member_cluster1, cluster1[i])\r\n        else:\r\n            member_cluster1 = list_math_addition(member_cluster1, data[ cluster1[i] ])\r\n\r\n    for j in range(len(cluster2)):\r\n        if data is None:\r\n            member_cluster2 = list_math_addition(member_cluster2, cluster2[j])\r\n        else:\r\n            member_cluster2 = list_math_addition(member_cluster2, data[ cluster2[j] ])\r\n    \r\n    member_cluster_general = list_math_addition(member_cluster1, member_cluster2)\r\n    member_cluster_general = list_math_division_number(member_cluster_general, len(cluster1) + len(cluster2))\r\n    \r\n    member_cluster1 = list_math_division_number(member_cluster1, len(cluster1))\r\n    member_cluster2 = list_math_division_number(member_cluster2, len(cluster2))\r\n    \r\n    # calculate global sum\r\n    distance_general = 0.0\r\n    distance_cluster1 = 0.0\r\n    distance_cluster2 = 0.0\r\n    \r\n    for i in range(len(cluster1)):\r\n        if data is None:\r\n            distance_cluster1 += euclidean_distance_square(cluster1[i], member_cluster1)\r\n            distance_general += euclidean_distance_square(cluster1[i], member_cluster_general)\r\n            \r\n        else:\r\n            distance_cluster1 += euclidean_distance_square(data[ cluster1[i]], member_cluster1)\r\n            distance_general += euclidean_distance_square(data[ cluster1[i]], member_cluster_general)\r\n    \r\n    for j in range(len(cluster2)):\r\n        if data is None:\r\n            distance_cluster2 += euclidean_distance_square(cluster2[j], member_cluster2)\r\n            distance_general += euclidean_distance_square(cluster2[j], member_cluster_general)\r\n            \r\n        else:\r\n            distance_cluster2 += euclidean_distance_square(data[ cluster2[j]], member_cluster2)\r\n            distance_general += euclidean_distance_square(data[ cluster2[j]], member_cluster_general)\r\n    \r\n    return distance_general - distance_cluster1 - distance_cluster2"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef calculate_ellipse_description(covariance, scale = 2.0):\r\n    \r\n    eigh_values, eigh_vectors = numpy.linalg.eigh(covariance)\r\n    order = eigh_values.argsort()[::-1]\r\n    \r\n    values, vectors = eigh_values[order], eigh_vectors[order]\r\n    angle = numpy.degrees(numpy.arctan2(*vectors[:,0][::-1]))\r\n\r\n    if 0.0 in values:\r\n        return 0, 0, 0\r\n\r\n    width, height = 2.0 * scale * numpy.sqrt(values)\r\n    return angle, width, height", "response": "Calculates the description of an ellipse using a covariance matrix."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef data_corners(data, data_filter = None):\r\n    \r\n    dimensions = len(data[0])\r\n    \r\n    bypass = data_filter\r\n    if bypass is None:\r\n        bypass = range(len(data))\r\n    \r\n    maximum_corner = list(data[bypass[0]][:])\r\n    minimum_corner = list(data[bypass[0]][:])\r\n    \r\n    for index_point in bypass:\r\n        for index_dimension in range(dimensions):\r\n            if data[index_point][index_dimension] > maximum_corner[index_dimension]:\r\n                maximum_corner[index_dimension] = data[index_point][index_dimension]\r\n            \r\n            if data[index_point][index_dimension] < minimum_corner[index_dimension]:\r\n                minimum_corner[index_dimension] = data[index_point][index_dimension]\r\n    \r\n    return minimum_corner, maximum_corner", "response": "Returns a tuple of two points that corresponds to the maximum and minimum corner in each dimension of the specified data."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncalculate the norm of an input vector that is known as a vector length.", "response": "def norm_vector(vector):\r\n    \"\"\"!\r\n    @brief Calculates norm of an input vector that is known as a vector length.\r\n    \r\n    @param[in] vector (list): The input vector whose length is calculated.\r\n    \r\n    @return (double) vector norm known as vector length.\r\n    \r\n    \"\"\"\r\n    \r\n    length = 0.0\r\n    for component in vector:\r\n        length += component * component\r\n    \r\n    length = length ** 0.5\r\n    \r\n    return length"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nexecuting specified function or method with measuring of execution time.", "response": "def timedcall(executable_function, *args):\r\n    \"\"\"!\r\n    @brief Executes specified method or function with measuring of execution time.\r\n    \r\n    @param[in] executable_function (pointer): Pointer to function or method.\r\n    @param[in] args (*): Arguments of called function or method.\r\n    \r\n    @return (tuple) Execution time and result of execution of function or method (execution_time, result_execution).\r\n    \r\n    \"\"\"\r\n    \r\n    time_start = time.clock();\r\n    result = executable_function(*args);\r\n    time_end = time.clock();\r\n    \r\n    return (time_end - time_start, result);"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef extract_number_oscillations(osc_dyn, index = 0, amplitude_threshold = 1.0):\r\n    \r\n    number_oscillations = 0;\r\n    waiting_differential = False;\r\n    threshold_passed = False;\r\n    high_level_trigger = True if (osc_dyn[0][index] > amplitude_threshold) else False;\r\n    \r\n    for values in osc_dyn:\r\n        if ( (values[index] >= amplitude_threshold) and (high_level_trigger is False) ):\r\n            high_level_trigger = True;\r\n            threshold_passed = True;\r\n        \r\n        elif ( (values[index] < amplitude_threshold) and (high_level_trigger is True) ):\r\n            high_level_trigger = False;\r\n            threshold_passed = True;\r\n        \r\n        if (threshold_passed is True):\r\n            threshold_passed = False;\r\n            if (waiting_differential is True and high_level_trigger is False):\r\n                number_oscillations += 1;\r\n                waiting_differential = False;\r\n\r\n            else:\r\n                waiting_differential = True;\r\n        \r\n    return number_oscillations;", "response": "Returns the number of oscillations in a dynamic list."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef allocate_sync_ensembles(dynamic, tolerance = 0.1, threshold = 1.0, ignore = None):\r\n    \r\n    descriptors = [ [] for _ in range(len(dynamic[0])) ];\r\n    \r\n    # Check from the end for obtaining result\r\n    for index_dyn in range(0, len(dynamic[0]), 1):\r\n        if ((ignore is not None) and (index_dyn in ignore)):\r\n            continue;\r\n        \r\n        time_stop_simulation = len(dynamic) - 1;\r\n        active_state = False;\r\n        \r\n        if (dynamic[time_stop_simulation][index_dyn] > threshold):\r\n            active_state = True;\r\n            \r\n        # if active state is detected, it means we don't have whole oscillatory period for the considered oscillator, should be skipped.\r\n        if (active_state is True):\r\n            while ( (dynamic[time_stop_simulation][index_dyn] > threshold) and (time_stop_simulation > 0) ):\r\n                time_stop_simulation -= 1;\r\n            \r\n            # if there are no any oscillation than let's consider it like noise\r\n            if (time_stop_simulation == 0):\r\n                continue;\r\n            \r\n            # reset\r\n            active_state = False;\r\n        \r\n        desc = [0, 0, 0]; # end, start, average time of oscillation\r\n        for t in range(time_stop_simulation, 0, -1):\r\n            if ( (dynamic[t][index_dyn] > threshold) and (active_state is False) ):\r\n                desc[0] = t;\r\n                active_state = True;\r\n            elif ( (dynamic[t][index_dyn] < threshold) and (active_state is True) ):\r\n                desc[1] = t;\r\n                active_state = False;\r\n                \r\n                break;\r\n        \r\n        if (desc == [0, 0, 0]):\r\n            continue;\r\n        \r\n        desc[2] = desc[1] + (desc[0] - desc[1]) / 2.0;\r\n        descriptors[index_dyn] = desc;\r\n    \r\n    \r\n    # Cluster allocation\r\n    sync_ensembles = [];\r\n    desc_sync_ensembles = [];\r\n    \r\n    for index_desc in range(0, len(descriptors), 1):\r\n        if (descriptors[index_desc] == []):\r\n            continue;\r\n        \r\n        if (len(sync_ensembles) == 0):\r\n            desc_ensemble = descriptors[index_desc];\r\n            reducer = (desc_ensemble[0] - desc_ensemble[1]) * tolerance;\r\n            \r\n            desc_ensemble[0] = desc_ensemble[2] + reducer;\r\n            desc_ensemble[1] = desc_ensemble[2] - reducer;\r\n            \r\n            desc_sync_ensembles.append(desc_ensemble);\r\n            sync_ensembles.append([ index_desc ]);\r\n        else:\r\n            oscillator_captured = False;\r\n            for index_ensemble in range(0, len(sync_ensembles), 1):\r\n                if ( (desc_sync_ensembles[index_ensemble][0] > descriptors[index_desc][2]) and (desc_sync_ensembles[index_ensemble][1] < descriptors[index_desc][2])):\r\n                    sync_ensembles[index_ensemble].append(index_desc);\r\n                    oscillator_captured = True;\r\n                    break;\r\n                \r\n            if (oscillator_captured is False):\r\n                desc_ensemble = descriptors[index_desc];\r\n                reducer = (desc_ensemble[0] - desc_ensemble[1]) * tolerance;\r\n        \r\n                desc_ensemble[0] = desc_ensemble[2] + reducer;\r\n                desc_ensemble[1] = desc_ensemble[2] - reducer;\r\n        \r\n                desc_sync_ensembles.append(desc_ensemble);\r\n                sync_ensembles.append([ index_desc ]);\r\n    \r\n    return sync_ensembles;", "response": "Allocate clusters in line with ensembles of synchronous oscillators that are not in the same sync ensemble."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef draw_clusters(data, clusters, noise = [], marker_descr = '.', hide_axes = False, axes = None, display_result = True):\r\n    # Get dimension\r\n    dimension = 0;\r\n    if ( (data is not None) and (clusters is not None) ):\r\n        dimension = len(data[0]);\r\n    elif ( (data is None) and (clusters is not None) ):\r\n        dimension = len(clusters[0][0]);\r\n    else:\r\n        raise NameError('Data or clusters should be specified exactly.');\r\n    \r\n    \"Draw clusters\"\r\n    colors = [ 'red', 'blue', 'darkgreen', 'brown', 'violet', \r\n               'deepskyblue', 'darkgrey', 'lightsalmon', 'deeppink', 'yellow',\r\n               'black', 'mediumspringgreen', 'orange', 'darkviolet', 'darkblue',\r\n               'silver', 'lime', 'pink', 'gold', 'bisque' ];\r\n               \r\n    if (len(clusters) > len(colors)):\r\n        raise NameError('Impossible to represent clusters due to number of specified colors.');\r\n    \r\n    fig = plt.figure();\r\n    \r\n    if (axes is None):\r\n        # Check for dimensions\r\n        if ((dimension) == 1 or (dimension == 2)):\r\n            axes = fig.add_subplot(111);\r\n        elif (dimension == 3):\r\n            axes = fig.gca(projection='3d');\r\n        else:\r\n            raise NameError('Drawer supports only 2d and 3d data representation');\r\n    \r\n    color_index = 0;\r\n    for cluster in clusters:\r\n        color = colors[color_index];\r\n        for item in cluster:\r\n            if (dimension == 1):\r\n                if (data is None):\r\n                    axes.plot(item[0], 0.0, color = color, marker = marker_descr);\r\n                else:\r\n                    axes.plot(data[item][0], 0.0, color = color, marker = marker_descr);\r\n            \r\n            if (dimension == 2):\r\n                if (data is None):\r\n                    axes.plot(item[0], item[1], color = color, marker = marker_descr);\r\n                else:\r\n                    axes.plot(data[item][0], data[item][1], color = color, marker = marker_descr);\r\n                    \r\n            elif (dimension == 3):\r\n                if (data is None):\r\n                    axes.scatter(item[0], item[1], item[2], c = color, marker = marker_descr);\r\n                else:\r\n                    axes.scatter(data[item][0], data[item][1], data[item][2], c = color, marker = marker_descr);\r\n        \r\n        color_index += 1;\r\n    \r\n    for item in noise:\r\n        if (dimension == 1):\r\n            if (data is None):\r\n                axes.plot(item[0], 0.0, 'w' + marker_descr);\r\n            else:\r\n                axes.plot(data[item][0], 0.0, 'w' + marker_descr);\r\n\r\n        if (dimension == 2):\r\n            if (data is None):\r\n                axes.plot(item[0], item[1], 'w' + marker_descr);\r\n            else:\r\n                axes.plot(data[item][0], data[item][1], 'w' + marker_descr);\r\n                \r\n        elif (dimension == 3):\r\n            if (data is None):\r\n                axes.scatter(item[0], item[1], item[2], c = 'w', marker = marker_descr);\r\n            else:\r\n                axes.scatter(data[item][0], data[item][1], data[item][2], c = 'w', marker = marker_descr);\r\n    \r\n    axes.grid(True);\r\n    \r\n    if (hide_axes is True):\r\n        axes.xaxis.set_ticklabels([]);\r\n        axes.yaxis.set_ticklabels([]);\r\n        \r\n        if (dimension == 3):\r\n            axes.zaxis.set_ticklabels([]);\r\n    \r\n    if (display_result is True):\r\n        plt.show();\r\n\r\n    return axes;", "response": "Displays clusters for data in 2D or 3D or 3D format."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nsetting the parameters for a single matplotlib axes.", "response": "def set_ax_param(ax, x_title = None, y_title = None, x_lim = None, y_lim = None, x_labels = True, y_labels = True, grid = True):\r\n    \"\"\"!\r\n    @brief Sets parameters for matplotlib ax.\r\n    \r\n    @param[in] ax (Axes): Axes for which parameters should applied.\r\n    @param[in] x_title (string): Title for Y.\r\n    @param[in] y_title (string): Title for X.\r\n    @param[in] x_lim (double): X limit.\r\n    @param[in] y_lim (double): Y limit.\r\n    @param[in] x_labels (bool): If True - shows X labels.\r\n    @param[in] y_labels (bool): If True - shows Y labels.\r\n    @param[in] grid (bool): If True - shows grid.\r\n    \r\n    \"\"\"\r\n    from matplotlib.font_manager import FontProperties;\r\n    from matplotlib import rcParams;\r\n    \r\n    if (_platform == \"linux\") or (_platform == \"linux2\"):\r\n        rcParams['font.sans-serif'] = ['Liberation Serif'];\r\n    else:\r\n        rcParams['font.sans-serif'] = ['Arial'];\r\n        \r\n    rcParams['font.size'] = 12;\r\n        \r\n    surface_font = FontProperties();\r\n    if (_platform == \"linux\") or (_platform == \"linux2\"):\r\n        surface_font.set_name('Liberation Serif');\r\n    else:\r\n        surface_font.set_name('Arial');\r\n        \r\n    surface_font.set_size('12');\r\n    \r\n    if (y_title is not None): ax.set_ylabel(y_title, fontproperties = surface_font);\r\n    if (x_title is not None): ax.set_xlabel(x_title, fontproperties = surface_font);\r\n    \r\n    if (x_lim is not None): ax.set_xlim(x_lim[0], x_lim[1]);\r\n    if (y_lim is not None): ax.set_ylim(y_lim[0], y_lim[1]);\r\n    \r\n    if (x_labels is False): ax.xaxis.set_ticklabels([]);\r\n    if (y_labels is False): ax.yaxis.set_ticklabels([]);\r\n    \r\n    ax.grid(grid);"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef draw_dynamics_set(dynamics, xtitle = None, ytitle = None, xlim = None, ylim = None, xlabels = False, ylabels = False):\r\n    # Calculate edge for confortable representation.\r\n    number_dynamics = len(dynamics);\r\n    if (number_dynamics == 1):\r\n        draw_dynamics(dynamics[0][0], dynamics[0][1], xtitle, ytitle, xlim, ylim, xlabels, ylabels);\r\n        return;\r\n    \r\n    number_cols = int(numpy.ceil(number_dynamics ** 0.5));\r\n    number_rows = int(numpy.ceil(number_dynamics / number_cols));\r\n\r\n    real_index = 0, 0;\r\n    double_indexer = True;\r\n    if ( (number_cols == 1) or (number_rows == 1) ):\r\n        real_index = 0;\r\n        double_indexer = False;\r\n    \r\n    (_, axarr) = plt.subplots(number_rows, number_cols);\r\n    #plt.setp([ax for ax in axarr], visible = False);\r\n    \r\n    for dynamic in dynamics:\r\n        axarr[real_index] = draw_dynamics(dynamic[0], dynamic[1], xtitle, ytitle, xlim, ylim, xlabels, ylabels, axes = axarr[real_index]);\r\n        #plt.setp(axarr[real_index], visible = True);\r\n        \r\n        if (double_indexer is True):\r\n            real_index = real_index[0], real_index[1] + 1;\r\n            if (real_index[1] >= number_cols):\r\n                real_index = real_index[0] + 1, 0; \r\n        else:\r\n            real_index += 1;\r\n            \r\n    plt.show();", "response": "Draws a set of dynamics in the network."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\ndisplays image segments using colored image.", "response": "def draw_image_color_segments(source, clusters, hide_axes = True):\r\n    \"\"\"!\r\n    @brief Shows image segments using colored image.\r\n    @details Each color on result image represents allocated segment. The first image is initial and other is result of segmentation.\r\n    \r\n    @param[in] source (string): Path to image.\r\n    @param[in] clusters (list): List of clusters (allocated segments of image) where each cluster\r\n                                consists of indexes of pixel from source image.\r\n    @param[in] hide_axes (bool): If True then axes will not be displayed.\r\n    \r\n    \"\"\"\r\n        \r\n    image_source = Image.open(source);\r\n    image_size = image_source.size;\r\n    \r\n    (fig, axarr) = plt.subplots(1, 2);\r\n    \r\n    plt.setp([ax for ax in axarr], visible = False);\r\n    \r\n    available_colors = [ (0, 162, 232),   (34, 177, 76),   (237, 28, 36),\r\n                         (255, 242, 0),   (0, 0, 0),       (237, 28, 36),\r\n                         (255, 174, 201), (127, 127, 127), (185, 122, 87), \r\n                         (200, 191, 231), (136, 0, 21),    (255, 127, 39),\r\n                         (63, 72, 204),   (195, 195, 195), (255, 201, 14),\r\n                         (239, 228, 176), (181, 230, 29),  (153, 217, 234),\r\n                         (112, 146, 180) ];\r\n    \r\n    image_color_segments = [(255, 255, 255)] * (image_size[0] * image_size[1]);\r\n    \r\n    for index_segment in range(len(clusters)):\r\n        for index_pixel in clusters[index_segment]:\r\n            image_color_segments[index_pixel] = available_colors[index_segment];\r\n    \r\n    stage = array(image_color_segments, numpy.uint8);\r\n    stage = numpy.reshape(stage, (image_size[1], image_size[0]) + ((3),)); # ((3),) it's size of RGB - third dimension.\r\n    image_cluster = Image.fromarray(stage, 'RGB');\r\n    \r\n    axarr[0].imshow(image_source, interpolation = 'none');\r\n    axarr[1].imshow(image_cluster, interpolation = 'none');\r\n    \r\n    for i in range(2):\r\n        plt.setp(axarr[i], visible = True);\r\n        \r\n        if (hide_axes is True):\r\n            axarr[i].xaxis.set_ticklabels([]);\r\n            axarr[i].yaxis.set_ticklabels([]);\r\n            axarr[i].xaxis.set_ticks_position('none');\r\n            axarr[i].yaxis.set_ticks_position('none');\r\n    \r\n    plt.show();"}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndrawing black masks of segments from a source image.", "response": "def draw_image_mask_segments(source, clusters, hide_axes = True):\r\n    \"\"\"!\r\n    @brief Shows image segments using black masks.\r\n    @details Each black mask of allocated segment is presented on separate plot.\r\n             The first image is initial and others are black masks of segments.\r\n    \r\n    @param[in] source (string): Path to image.\r\n    @param[in] clusters (list): List of clusters (allocated segments of image) where each cluster\r\n                                consists of indexes of pixel from source image.\r\n    @param[in] hide_axes (bool): If True then axes will not be displayed.\r\n    \r\n    \"\"\"\r\n    if (len(clusters) == 0):\r\n        print(\"Warning: Nothing to draw - list of clusters is empty.\")\r\n        return;\r\n        \r\n    image_source = Image.open(source);\r\n    image_size = image_source.size;\r\n    \r\n    # Calculate edge for confortable representation.\r\n    number_clusters = len(clusters) + 1; # show with the source image\r\n    \r\n    number_cols = int(numpy.ceil(number_clusters ** 0.5));\r\n    number_rows = int(numpy.ceil(number_clusters / number_cols));\r\n    \r\n\r\n    real_index = 0, 0;\r\n    double_indexer = True;\r\n    if ( (number_cols == 1) or (number_rows == 1) ):\r\n        real_index = 0;\r\n        double_indexer = False;\r\n    \r\n    (fig, axarr) = plt.subplots(number_rows, number_cols);\r\n    plt.setp([ax for ax in axarr], visible = False);\r\n    \r\n    axarr[real_index].imshow(image_source, interpolation = 'none');\r\n    plt.setp(axarr[real_index], visible = True);\r\n    \r\n    if (hide_axes is True):\r\n        axarr[real_index].xaxis.set_ticklabels([]);\r\n        axarr[real_index].yaxis.set_ticklabels([]);\r\n        axarr[real_index].xaxis.set_ticks_position('none');\r\n        axarr[real_index].yaxis.set_ticks_position('none');\r\n            \r\n    if (double_indexer is True):\r\n        real_index = 0, 1;\r\n    else:\r\n        real_index += 1;\r\n    \r\n    for cluster in clusters:\r\n        stage_cluster = [(255, 255, 255)] * (image_size[0] * image_size[1]);\r\n        for index in cluster:\r\n            stage_cluster[index] = (0, 0, 0);\r\n          \r\n        stage = array(stage_cluster, numpy.uint8);\r\n        stage = numpy.reshape(stage, (image_size[1], image_size[0]) + ((3),)); # ((3),) it's size of RGB - third dimension.\r\n        \r\n        image_cluster = Image.fromarray(stage, 'RGB');\r\n        \r\n        axarr[real_index].imshow(image_cluster, interpolation = 'none');\r\n        plt.setp(axarr[real_index], visible = True);\r\n        \r\n        if (hide_axes is True):\r\n            axarr[real_index].xaxis.set_ticklabels([]);\r\n            axarr[real_index].yaxis.set_ticklabels([]);\r\n            \r\n            axarr[real_index].xaxis.set_ticks_position('none');\r\n            axarr[real_index].yaxis.set_ticks_position('none');\r\n        \r\n        if (double_indexer is True):\r\n            real_index = real_index[0], real_index[1] + 1;\r\n            if (real_index[1] >= number_cols):\r\n                real_index = real_index[0] + 1, 0; \r\n        else:\r\n            real_index += 1;\r\n\r\n            \r\n    plt.show();"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalculates the linear sum of the input list that can be represented by list in case of multidimensional elements.", "response": "def linear_sum(list_vector):\r\n    \"\"\"!\r\n    @brief Calculates linear sum of vector that is represented by list, each element can be represented by list - multidimensional elements.\r\n    \r\n    @param[in] list_vector (list): Input vector.\r\n    \r\n    @return (list|double) Linear sum of vector that can be represented by list in case of multidimensional elements.\r\n    \r\n    \"\"\"\r\n    dimension = 1;\r\n    linear_sum = 0.0;\r\n    list_representation = (type(list_vector[0]) == list);\r\n    \r\n    if (list_representation is True):\r\n        dimension = len(list_vector[0]);\r\n        linear_sum = [0] * dimension;\r\n        \r\n    for index_element in range(0, len(list_vector)):\r\n        if (list_representation is True):\r\n            for index_dimension in range(0, dimension):\r\n                linear_sum[index_dimension] += list_vector[index_element][index_dimension];\r\n        else:\r\n            linear_sum += list_vector[index_element];\r\n\r\n    return linear_sum;"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef square_sum(list_vector):\r\n    \r\n    square_sum = 0.0;\r\n    list_representation = (type(list_vector[0]) == list);\r\n        \r\n    for index_element in range(0, len(list_vector)):\r\n        if (list_representation is True):\r\n            square_sum += sum(list_math_multiplication(list_vector[index_element], list_vector[index_element]));\r\n        else:\r\n            square_sum += list_vector[index_element] * list_vector[index_element];\r\n         \r\n    return square_sum;", "response": "Calculates the square sum of the list - multidimensional elements."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nreturn the number of mathematical subtraction between list a and b.", "response": "def list_math_substraction_number(a, b):\r\n    \"\"\"!\r\n    @brief Calculates subtraction between list and number.\r\n    @details Each element from list 'a' is subtracted by number 'b'.\r\n    \r\n    @param[in] a (list): List of elements that supports mathematical subtraction.\r\n    @param[in] b (list): Value that supports mathematical subtraction.\r\n    \r\n    @return (list) Results of subtraction between list and number.\r\n    \r\n    \"\"\"        \r\n    return [a[i] - b for i in range(len(a))];"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef list_math_addition(a, b):\r\n    return [a[i] + b[i] for i in range(len(a))];", "response": "Adds two lists of mathematic elements to the end of a list."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef list_math_addition_number(a, b):\r\n    return [a[i] + b for i in range(len(a))];", "response": "Adds two lists of mathematic addition numbers."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning a list of numbers that are divided by number b.", "response": "def list_math_division_number(a, b):\r\n    \"\"\"!\r\n    @brief Division between list and number.\r\n    @details Each element from list 'a' is divided by number 'b'.\r\n    \r\n    @param[in] a (list): List of elements that supports mathematic division.\r\n    @param[in] b (double): Value that supports mathematic division.\r\n    \r\n    @return (list) Result of division between list and number.\r\n    \r\n    \"\"\"    \r\n    return [a[i] / b for i in range(len(a))];"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef list_math_division(a, b):\r\n    return [a[i] / b[i] for i in range(len(a))];", "response": "Returns a list of mathematic division of two lists."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef list_math_multiplication_number(a, b):\r\n    return [a[i] * b for i in range(len(a))];", "response": "Returns a list that supports mathematic division between list a and number b."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef list_math_multiplication(a, b):\r\n    return [a[i] * b[i] for i in range(len(a))];", "response": "Returns the mathematic multiplication of two lists."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncreate all the oscillators and the user s entries between all the oscillators.", "response": "def __create_all_to_all_connections(self):\r\n        \"\"\"!\r\n        @brief Creates connections between all oscillators.\r\n        \r\n        \"\"\"\r\n        \r\n        if (self._conn_represent == conn_represent.MATRIX):\r\n            for index in range(0, self._num_osc, 1):\r\n                self._osc_conn.append([True] * self._num_osc);\r\n                self._osc_conn[index][index] = False;\r\n        \r\n        elif (self._conn_represent == conn_represent.LIST):\r\n            for index in range(0, self._num_osc, 1):\r\n                self._osc_conn.append([neigh for neigh in range(0, self._num_osc, 1) if index != neigh]);"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate four grid structure connections.", "response": "def __create_grid_four_connections(self):\r\n        \"\"\"!\r\n        @brief Creates network with connections that make up four grid structure.\r\n        @details Each oscillator may be connected with four neighbors in line with 'grid' structure: right, upper, left, lower.\r\n        \r\n        \"\"\"\r\n        \r\n        side_size = self.__width;\r\n        if (self._conn_represent == conn_represent.MATRIX):\r\n            self._osc_conn = [[0] * self._num_osc for index in range(0, self._num_osc, 1)];\r\n        elif (self._conn_represent == conn_represent.LIST):\r\n            self._osc_conn = [[] for index in range(0, self._num_osc, 1)];\r\n        else:\r\n            raise NameError(\"Unknown type of representation of connections\");\r\n        \r\n        for index in range(0, self._num_osc, 1):\r\n            upper_index = index - side_size;\r\n            lower_index = index + side_size;\r\n            left_index = index - 1;\r\n            right_index = index + 1;\r\n            \r\n            node_row_index = math.ceil(index / side_size);\r\n            if (upper_index >= 0):\r\n                self.__create_connection(index, upper_index);\r\n            \r\n            if (lower_index < self._num_osc):\r\n                self.__create_connection(index, lower_index);\r\n            \r\n            if ( (left_index >= 0) and (math.ceil(left_index / side_size) == node_row_index) ):\r\n                self.__create_connection(index, left_index);\r\n            \r\n            if ( (right_index < self._num_osc) and (math.ceil(right_index / side_size) == node_row_index) ):\r\n                self.__create_connection(index, right_index);"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncreates network with eight neighbors.", "response": "def __create_grid_eight_connections(self):\r\n        \"\"\"!\r\n        @brief Creates network with connections that make up eight grid structure.\r\n        @details Each oscillator may be connected with eight neighbors in line with grid structure: right, right-upper, upper, upper-left, left, left-lower, lower, lower-right.\r\n        \r\n        \"\"\"\r\n        \r\n        self.__create_grid_four_connections();     # create connection with right, upper, left, lower.\r\n        side_size = self.__width;\r\n        \r\n        for index in range(0, self._num_osc, 1):\r\n            upper_left_index = index - side_size - 1;\r\n            upper_right_index = index - side_size + 1;\r\n            \r\n            lower_left_index = index + side_size - 1;\r\n            lower_right_index = index + side_size + 1;\r\n            \r\n            node_row_index = math.floor(index / side_size);\r\n            upper_row_index = node_row_index - 1;\r\n            lower_row_index = node_row_index + 1;\r\n            \r\n            if ( (upper_left_index >= 0) and (math.floor(upper_left_index / side_size) == upper_row_index) ):\r\n                self.__create_connection(index, upper_left_index);\r\n            \r\n            if ( (upper_right_index >= 0) and (math.floor(upper_right_index / side_size) == upper_row_index) ):\r\n                self.__create_connection(index, upper_right_index);\r\n                \r\n            if ( (lower_left_index < self._num_osc) and (math.floor(lower_left_index / side_size) == lower_row_index) ):\r\n                self.__create_connection(index, lower_left_index);\r\n                \r\n            if ( (lower_right_index < self._num_osc) and (math.floor(lower_right_index / side_size) == lower_row_index) ):\r\n                self.__create_connection(index, lower_right_index);"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __create_list_bidir_connections(self):\r\n        \r\n        if (self._conn_represent == conn_represent.MATRIX):\r\n            for index in range(0, self._num_osc, 1):\r\n                self._osc_conn.append([0] * self._num_osc);\r\n                self._osc_conn[index][index] = False;\r\n                if (index > 0):\r\n                    self._osc_conn[index][index - 1] = True;\r\n                    \r\n                if (index < (self._num_osc - 1)):\r\n                    self._osc_conn[index][index + 1] = True;\r\n        \r\n        elif (self._conn_represent == conn_represent.LIST):\r\n            for index in range(self._num_osc):\r\n                self._osc_conn.append([]);\r\n                if (index > 0):\r\n                    self._osc_conn[index].append(index - 1);\r\n                \r\n                if (index < (self._num_osc - 1)):\r\n                    self._osc_conn[index].append(index + 1);", "response": "Creates the list of classical list connections."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __create_none_connections(self):\r\n        if (self._conn_represent == conn_represent.MATRIX):\r\n            for _ in range(0, self._num_osc, 1):\r\n                self._osc_conn.append([False] * self._num_osc);\r\n        elif (self._conn_represent == conn_represent.LIST):\r\n            self._osc_conn = [[] for _ in range(0, self._num_osc, 1)];", "response": "Creates network with no connections."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncreates the connection in line with representation of matrix connections.", "response": "def _create_structure(self, type_conn = conn_type.ALL_TO_ALL):\r\n        \"\"\"!\r\n        @brief Creates connection in line with representation of matrix connections [NunOsc x NumOsc].\r\n        \r\n        @param[in] type_conn (conn_type): Connection type (all-to-all, bidirectional list, grid structure, etc.) that is used by the network.\r\n        \r\n        \"\"\"\r\n        \r\n        self._osc_conn = list();\r\n        \r\n        if (type_conn == conn_type.NONE):\r\n            self.__create_none_connections();\r\n        \r\n        elif (type_conn == conn_type.ALL_TO_ALL):\r\n            self.__create_all_to_all_connections();\r\n        \r\n        elif (type_conn == conn_type.GRID_FOUR):\r\n            self.__create_grid_four_connections();\r\n            \r\n        elif (type_conn == conn_type.GRID_EIGHT):\r\n            self.__create_grid_eight_connections();\r\n            \r\n        elif (type_conn == conn_type.LIST_BIDIR):\r\n            self.__create_list_bidir_connections();\r\n        \r\n        elif (type_conn == conn_type.DYNAMIC):\r\n            self.__create_dynamic_connection();\r\n        \r\n        else:\r\n            raise NameError('The unknown type of connections');"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nreturning True if there is a connection between i and j oscillators and False otherwise.", "response": "def has_connection(self, i, j):\r\n        \"\"\"!\r\n        @brief Returns True if there is connection between i and j oscillators and False - if connection doesn't exist.\r\n        \r\n        @param[in] i (uint): index of an oscillator in the network.\r\n        @param[in] j (uint): index of an oscillator in the network.\r\n        \r\n        \"\"\"\r\n        if (self._conn_represent == conn_represent.MATRIX):\r\n            return (self._osc_conn[i][j]);\r\n        \r\n        elif (self._conn_represent == conn_represent.LIST):\r\n            for neigh_index in range(0, len(self._osc_conn[i]), 1):\r\n                if (self._osc_conn[i][neigh_index] == j):\r\n                    return True;\r\n            return False;\r\n        \r\n        else:\r\n            raise NameError(\"Unknown type of representation of coupling\");"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef set_connection(self, i, j):\r\n        \r\n        if (self.structure != conn_type.DYNAMIC):\r\n            raise NameError(\"Connection between oscillators can be changed only in case of dynamic type.\");\r\n        \r\n        if (self._conn_represent == conn_represent.MATRIX):\r\n            self._osc_conn[i][j] = True;\r\n            self._osc_conn[j][i] = True;\r\n        else:\r\n            self._osc_conn[i].append(j);\r\n            self._osc_conn[j].append(i);", "response": "Sets the connection between two specified oscillators."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef get_neighbors(self, index):\r\n        \r\n        if (self._conn_represent == conn_represent.LIST):\r\n            return self._osc_conn[index];      # connections are represented by list.\r\n        elif (self._conn_represent == conn_represent.MATRIX):\r\n            return [neigh_index for neigh_index in range(self._num_osc) if self._osc_conn[index][neigh_index] == True];\r\n        else:\r\n            raise NameError(\"Unknown type of representation of connections\");", "response": "Returns a list of neighbors of the specified oscillator with the specified index."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef process(self):\r\n        \r\n        random.seed()\r\n        \r\n        for _ in range(0, self.__numlocal):\r\n            # set (current) random medoids\r\n            self.__current = random.sample(range(0, len(self.__pointer_data)), self.__number_clusters)\r\n            \r\n            # update clusters in line with random allocated medoids\r\n            self.__update_clusters(self.__current)\r\n            \r\n            # optimize configuration\r\n            self.__optimize_configuration()\r\n            \r\n            # obtain cost of current cluster configuration and compare it with the best obtained\r\n            estimation = self.__calculate_estimation()\r\n            if estimation < self.__optimal_estimation:\r\n                self.__optimal_medoids = self.__current[:]\r\n                self.__optimal_estimation = estimation\r\n        \r\n        self.__update_clusters(self.__optimal_medoids)", "response": "This method is called by the process method in order to process the current cluster."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __update_clusters(self, medoids):\r\n        \r\n        self.__belong = [0] * len(self.__pointer_data)\r\n        self.__clusters = [[] for i in range(len(medoids))]\r\n        for index_point in range(len(self.__pointer_data)):\r\n            index_optim = -1\r\n            dist_optim = 0.0\r\n             \r\n            for index in range(len(medoids)):\r\n                dist = euclidean_distance_square(self.__pointer_data[index_point], self.__pointer_data[medoids[index]])\r\n                 \r\n                if (dist < dist_optim) or (index is 0):\r\n                    index_optim = index\r\n                    dist_optim = dist\r\n\r\n            self.__clusters[index_optim].append(index_point)\r\n            self.__belong[index_point] = index_optim\r\n        \r\n        # If cluster is not able to capture object it should be removed\r\n        self.__clusters = [cluster for cluster in self.__clusters if len(cluster) > 0]", "response": "Updates the list of clusters based on the pointer data."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __find_another_nearest_medoid(self, point_index, current_medoid_index):\r\n        other_medoid_index = -1\r\n        other_distance_nearest = float('inf')\r\n        for index_medoid in self.__current:\r\n            if (index_medoid != current_medoid_index):\r\n                other_distance_candidate = euclidean_distance_square(self.__pointer_data[point_index], self.__pointer_data[current_medoid_index])\r\n                \r\n                if other_distance_candidate < other_distance_nearest:\r\n                    other_distance_nearest = other_distance_candidate\r\n                    other_medoid_index = index_medoid\r\n        \r\n        return other_medoid_index", "response": "Finds the nearest medoid for the specified point that is differ from the specified medoid. Returns the index of the nearest medoid."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __calculate_estimation(self):\r\n        estimation = 0.0\r\n        for index_cluster in range(0, len(self.__clusters)):\r\n            cluster = self.__clusters[index_cluster]\r\n            index_medoid = self.__current[index_cluster]\r\n            for index_point in cluster:\r\n                estimation += euclidean_distance_square(self.__pointer_data[index_point], self.__pointer_data[index_medoid])\r\n        \r\n        return estimation", "response": "Calculates the estimation of the current clusters. The lower the estimation the more optimally configuration of clusters. The more optimally configuration of clusters is more optimally configuration of clusters."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef visualize(self, display=True):\r\n\r\n        kdnodes = self.__get_nodes()\r\n        level = kdnodes[0]\r\n        \r\n        for kdnode in kdnodes:\r\n            self.__print_node(level, kdnode)\r\n\r\n        self.__tree_text += self.__tree_level_text\r\n        if display is True:\r\n            print(self.__tree_text)\r\n        \r\n        return self.__tree_text", "response": "Display the KD - tree."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ninsert a new point with payload into the kd - tree.", "response": "def insert(self, point, payload):\r\n        \"\"\"!\r\n        @brief Insert new point with payload to kd-tree.\r\n        \r\n        @param[in] point (list): Coordinates of the point of inserted node.\r\n        @param[in] payload (any-type): Payload of inserted node. It can be identificator of the node or\r\n                    some useful payload that belongs to the point.\r\n        \r\n        @return (node) Inserted node to the kd-tree.\r\n        \r\n        \"\"\"\r\n        \r\n        if self.__root is None:\r\n            self.__dimension = len(point)\r\n            self.__root = node(point, payload, None, None, 0)\r\n            self.__point_comparator = self.__create_point_comparator(type(point))\r\n            return self.__root\r\n        \r\n        cur_node = self.__root\r\n        \r\n        while True:\r\n            if cur_node.data[cur_node.disc] <= point[cur_node.disc]:\r\n                # If new node is greater or equal than current node then check right leaf\r\n                if cur_node.right is None:\r\n                    discriminator = cur_node.disc + 1\r\n                    if discriminator >= self.__dimension:\r\n                        discriminator = 0\r\n                        \r\n                    cur_node.right = node(point, payload, None, None, discriminator, cur_node)\r\n                    return cur_node.right\r\n                else: \r\n                    cur_node = cur_node.right\r\n            \r\n            else:\r\n                # If new node is less than current then check left leaf\r\n                if cur_node.left is None:\r\n                    discriminator = cur_node.disc + 1\r\n                    if discriminator >= self.__dimension:\r\n                        discriminator = 0\r\n                        \r\n                    cur_node.left = node(point, payload, None, None, discriminator, cur_node)\r\n                    return cur_node.left\r\n                else:\r\n                    cur_node = cur_node.left"}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nremoves a node from the k - d tree.", "response": "def remove(self, point, **kwargs):\r\n        \"\"\"!\r\n        @brief Remove specified point from kd-tree.\r\n        @details It removes the first found node that satisfy to the input parameters. Make sure that\r\n                  pair (point, payload) is unique for each node, othewise the first found is removed.\r\n        \r\n        @param[in] point (list): Coordinates of the point of removed node.\r\n        @param[in] **kwargs: Arbitrary keyword arguments (available arguments: 'payload').\r\n        \r\n        <b>Keyword Args:</b><br>\r\n            - payload (any): Payload of the node that should be removed.\r\n        \r\n        @return (node) Root if node has been successfully removed, otherwise None.\r\n        \r\n        \"\"\"\r\n        \r\n        # Get required node\r\n        node_for_remove = None\r\n        if 'payload' in kwargs:\r\n            node_for_remove = self.find_node_with_payload(point, kwargs['payload'], None)\r\n        else:\r\n            node_for_remove = self.find_node(point, None)\r\n        \r\n        if node_for_remove is None:\r\n            return None\r\n        \r\n        parent = node_for_remove.parent\r\n        minimal_node = self.__recursive_remove(node_for_remove)\r\n        if parent is None:\r\n            self.__root = minimal_node\r\n            \r\n            # If all k-d tree was destroyed\r\n            if minimal_node is not None:\r\n                minimal_node.parent = None\r\n        else:\r\n            if parent.left is node_for_remove:\r\n                parent.left = minimal_node\r\n            elif parent.right is node_for_remove:\r\n                parent.right = minimal_node\r\n        \r\n        return self.__root"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef find_minimal_node(self, node_head, discriminator):\r\n        \r\n        min_key = lambda cur_node: cur_node.data[discriminator]\r\n        stack = []\r\n        candidates = []\r\n        isFinished = False\r\n        while isFinished is False:\r\n            if node_head is not None:\r\n                stack.append(node_head)\r\n                node_head = node_head.left\r\n            else:\r\n                if len(stack) != 0:\r\n                    node_head = stack.pop()\r\n                    candidates.append(node_head)\r\n                    node_head = node_head.right\r\n                else:\r\n                    isFinished = True\r\n\r\n        return min(candidates, key = min_key)", "response": "Find the minimal node in line with coordinate that is defined by discriminator."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nfills the KD - tree by specified data and create point comparator in line with data type.", "response": "def __fill_tree(self, data_list, payload_list):\r\n        \"\"\"!\r\n        @brief Fill KD-tree by specified data and create point comparator in line with data type.\r\n\r\n        @param[in] data_list (array_like): Data points that should be inserted to the tree.\r\n        @param[in] payload_list (array_like): Data point payloads that follows data points inserted to the tree.\r\n\r\n        \"\"\"\r\n        if data_list is None or len(data_list) == 0:\r\n            return # Just return from here, tree can be filled by insert method later\r\n\r\n        if payload_list is None:\r\n            # Case when payload is not specified.\r\n            for index in range(0, len(data_list)):\r\n                self.insert(data_list[index], None)\r\n        else:\r\n            # Case when payload is specified.\r\n            for index in range(0, len(data_list)):\r\n                self.insert(data_list[index], payload_list[index])\r\n\r\n        self.__point_comparator = self.__create_point_comparator(type(self.__root.data))"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __create_point_comparator(self, type_point):\r\n        if type_point == numpy.ndarray:\r\n            return lambda obj1, obj2: numpy.array_equal(obj1, obj2)\r\n\r\n        return lambda obj1, obj2: obj1 == obj2", "response": "Create a function that returns a function that compares objects with the point data type."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nsearch node that satisfies to input parameters in search rule.", "response": "def __find_node_by_rule(self, point, search_rule, cur_node):\r\n        \"\"\"!\r\n        @brief Search node that satisfy to parameters in search rule.\r\n        @details If node with specified parameters does not exist then None will be returned, \r\n                  otherwise required node will be returned.\r\n        \r\n        @param[in] point (list): Coordinates of the point whose node should be found.\r\n        @param[in] search_rule (lambda): Rule that is called to check whether node satisfies to search parameter.\r\n        @param[in] cur_node (node): Node from which search should be started.\r\n        \r\n        @return (node) Node if it satisfies to input parameters, otherwise it return None.\r\n        \r\n        \"\"\"\r\n        \r\n        req_node = None\r\n        \r\n        if cur_node is None:\r\n            cur_node = self.__root\r\n        \r\n        while cur_node:\r\n            if cur_node.data[cur_node.disc] <= point[cur_node.disc]:\r\n                # Check if it's required node\r\n                if search_rule(cur_node):\r\n                    req_node = cur_node\r\n                    break\r\n                \r\n                cur_node = cur_node.right\r\n            \r\n            else:\r\n                cur_node = cur_node.left\r\n        \r\n        return req_node"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef find_node_with_payload(self, point, point_payload, cur_node = None):\r\n        \r\n        rule_search = lambda node, point=point, payload=point_payload: self.__point_comparator(node.data, point) and node.payload == payload\r\n        return self.__find_node_by_rule(point, rule_search, cur_node)", "response": "Returns a node with the specified coordinates and payload."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef find_node(self, point, cur_node = None):\r\n        \r\n        rule_search = lambda node, point=point: self.__point_comparator(node.data, point)\r\n        return self.__find_node_by_rule(point, rule_search, cur_node)", "response": "Returns a node with coordinates that are defined by the specified point."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef find_nearest_dist_node(self, point, distance, retdistance = False):\r\n        \r\n        best_nodes = self.find_nearest_dist_nodes(point, distance)\r\n            \r\n        if best_nodes == []:\r\n            return None\r\n        \r\n        nearest = min(best_nodes, key = lambda item: item[0])\r\n        \r\n        if retdistance is True:\r\n            return nearest\r\n        else:\r\n            return nearest[1]", "response": "Find nearest neighbor in area with radius = distance."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef find_nearest_dist_nodes(self, point, distance):\r\n\r\n        best_nodes = []\r\n        if self.__root is not None:\r\n            self.__recursive_nearest_nodes(point, distance, distance * distance, self.__root, best_nodes)\r\n\r\n        return best_nodes", "response": "Find nearest neighbors in area that are covered by specified distance from the center of the area."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __recursive_nearest_nodes(self, point, distance, sqrt_distance, node_head, best_nodes):\r\n\r\n        if node_head.right is not None:\r\n            minimum = node_head.data[node_head.disc] - distance\r\n            if point[node_head.disc] >= minimum:\r\n                self.__recursive_nearest_nodes(point, distance, sqrt_distance, node_head.right, best_nodes)\r\n        \r\n        if node_head.left is not None:\r\n            maximum = node_head.data[node_head.disc] + distance\r\n            if point[node_head.disc] < maximum:\r\n                self.__recursive_nearest_nodes(point, distance, sqrt_distance, node_head.left, best_nodes)\r\n        \r\n        candidate_distance = euclidean_distance_square(point, node_head.data)\r\n        if candidate_distance <= sqrt_distance:\r\n            best_nodes.append( (candidate_distance, node_head) )", "response": "!\r\n        @brief Returns list of neighbors such as tuple (distance, node) that is located in area that is covered by distance.\r\n        \r\n        @param[in] point (list): Coordinates that is considered as centroind for searching\r\n        @param[in] distance (double): Distance from the center where seaching is performed.\r\n        @param[in] sqrt_distance (double): Square distance from the center where searching is performed.\r\n        @param[in] node_head (node): Node from that searching is performed.\r\n        @param[in|out] best_nodes (list): List of founded nodes."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef children(self, node_parent):\r\n        \r\n        if node_parent.left is not None:\r\n            yield node_parent.left\r\n        if node_parent.right is not None:\r\n            yield node_parent.right", "response": "Returns a list of children of a node."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning a list of tuples that are the items of the tree that are defined by the node specified by start_node and level.", "response": "def traverse(self, start_node = None, level = None):\r\n        \"\"\"!\r\n        @brief Traverses all nodes of subtree that is defined by node specified in input parameter.\r\n        \r\n        @param[in] start_node (node): Node from that travering of subtree is performed.\r\n        @param[in, out] level (uint): Should be ignored by application.\r\n        \r\n        @return (list) All nodes of the subtree.\r\n        \r\n        \"\"\"\r\n        \r\n        if start_node is None:\r\n            start_node  = self.__root\r\n            level = 0\r\n        \r\n        if start_node is None:\r\n            return []\r\n        \r\n        items = [ (level, start_node) ]\r\n        for child in self.children(start_node):\r\n            if child is not None:\r\n                items += self.traverse(child, level + 1)\r\n        \r\n        return items"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __process_by_ccore(self):\r\n        if isinstance(self.__initializer, kmeans_plusplus_initializer):\r\n            initializer = wrapper.elbow_center_initializer.KMEANS_PLUS_PLUS\r\n        else:\r\n            initializer = wrapper.elbow_center_initializer.RANDOM\r\n\r\n        result = wrapper.elbow(self.__data, self.__kmin, self.__kmax, initializer)\r\n\r\n        self.__kvalue = result[0]\r\n        self.__wce = result[1]", "response": "A method that processes the data from the ELBO and stores the result in the object."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __process_by_python(self):\r\n        for amount in range(self.__kmin, self.__kmax):\r\n            centers = self.__initializer(self.__data, amount).initialize()\r\n            instance = kmeans(self.__data, centers, ccore=True)\r\n            instance.process()\r\n\r\n            self.__wce.append(instance.get_total_wce())\r\n\r\n        self.__calculate_elbows()\r\n        self.__find_optimal_kvalue()", "response": "Performs processing using python implementation."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __calculate_elbows(self):\r\n\r\n        x0, y0 = 0.0, self.__wce[0]\r\n        x1, y1 = float(len(self.__wce)), self.__wce[-1]\r\n\r\n        for index_elbow in range(1, len(self.__wce) - 1):\r\n            x, y = float(index_elbow), self.__wce[index_elbow]\r\n\r\n            segment = abs((y0 - y1) * x + (x1 - x0) * y + (x0 * y1 - x1 * y0))\r\n            norm = math.sqrt((x1 - x0) ** 2 + (y1 - y0) ** 2)\r\n            distance = segment / norm\r\n\r\n            self.__elbows.append(distance)", "response": "!\r\n        @brief Calculates potential elbows.\r\n        @details Elbow is calculated as a distance from each point (x, y) to segment from kmin-point (x0, y0) to kmax-point (x1, y1)."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __find_optimal_kvalue(self):\r\n        optimal_elbow_value = max(self.__elbows)\r\n        self.__kvalue = self.__elbows.index(optimal_elbow_value) + 1 + self.__kmin", "response": "Find the optimal k - value for the current set of keys."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef calculate_connvectivity_radius(self, amount_clusters, maximum_iterations = 100):\r\n        \r\n        maximum_distance = max(self.__ordering)\r\n        \r\n        upper_distance = maximum_distance\r\n        lower_distance = 0.0\r\n\r\n        result = None\r\n        \r\n        amount, borders = self.extract_cluster_amount(maximum_distance)\r\n        if amount <= amount_clusters:\r\n            for _ in range(maximum_iterations):\r\n                radius = (lower_distance + upper_distance) / 2.0\r\n                \r\n                amount, borders = self.extract_cluster_amount(radius)\r\n                if amount == amount_clusters:\r\n                    result = radius\r\n                    break\r\n                \r\n                elif amount == 0:\r\n                    break\r\n                \r\n                elif amount > amount_clusters:\r\n                    lower_distance = radius\r\n                \r\n                elif amount < amount_clusters:\r\n                    upper_distance = radius\r\n        \r\n        return result, borders", "response": "Calculates the connectivity radius of allocation specified amount of clusters using the ordering diagram and indexes of values of values of values of ordering diagram."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef extract_cluster_amount(self, radius):\r\n        \r\n        amount_clusters = 1\r\n        \r\n        cluster_start = False\r\n        cluster_pick = False\r\n        total_similarity = True\r\n        previous_cluster_distance = None\r\n        previous_distance = None\r\n        \r\n        cluster_borders = []\r\n        \r\n        for index_ordering in range(len(self.__ordering)):\r\n            distance = self.__ordering[index_ordering]\r\n            if distance >= radius:\r\n                if cluster_start is False:\r\n                    cluster_start = True\r\n                    amount_clusters += 1\r\n                    \r\n                    if index_ordering != 0:\r\n                        cluster_borders.append(index_ordering)\r\n                \r\n                else:\r\n                    if (distance < previous_cluster_distance) and (cluster_pick is False):\r\n                        cluster_pick = True\r\n                    \r\n                    elif (distance > previous_cluster_distance) and (cluster_pick is True):\r\n                        cluster_pick = False\r\n                        amount_clusters += 1\r\n                        \r\n                        if index_ordering != 0:\r\n                            cluster_borders.append(index_ordering)\r\n                \r\n                previous_cluster_distance = distance\r\n            \r\n            else:\r\n                cluster_start = False\r\n                cluster_pick = False\r\n            \r\n            if (previous_distance is not None) and (distance != previous_distance):\r\n                total_similarity = False\r\n            \r\n            previous_distance = distance\r\n        \r\n        if (total_similarity is True) and (previous_distance > radius):\r\n            amount_clusters = 0\r\n\r\n        return amount_clusters, cluster_borders", "response": "Returns the amount of clusters allocated by the given radius on the ordering diagram."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __process_by_ccore(self):\r\n\r\n        (self.__clusters, self.__noise, self.__ordering, self.__eps,\r\n         objects_indexes, objects_core_distances, objects_reachability_distances) = \\\r\n            wrapper.optics(self.__sample_pointer, self.__eps, self.__minpts, self.__amount_clusters, self.__data_type)\r\n\r\n        self.__optics_objects = []\r\n        for i in range(len(objects_indexes)):\r\n            if objects_core_distances[i] < 0.0:\r\n                objects_core_distances[i] = None\r\n\r\n            if objects_reachability_distances[i] < 0.0:\r\n                objects_reachability_distances[i] = None\r\n\r\n            optics_object = optics_descriptor(objects_indexes[i], objects_core_distances[i], objects_reachability_distances[i])\r\n            optics_object.processed = True\r\n\r\n            self.__optics_objects.append(optics_object)", "response": "Processes the optics objects in the cluster by CCORE."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __initialize(self, sample):\r\n        \r\n        self.__processed = [False] * len(sample)\r\n        self.__optics_objects = [optics_descriptor(i) for i in range(len(sample))]      # List of OPTICS objects that corresponds to objects from input sample.\r\n        self.__ordered_database = []        # List of OPTICS objects in traverse order.\r\n        \r\n        self.__clusters = None      # Result of clustering (list of clusters where each cluster contains indexes of objects from input data).\r\n        self.__noise = None", "response": "Initializes internal states and resets clustering results in line with input sample."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __allocate_clusters(self):\r\n        \r\n        self.__initialize(self.__sample_pointer)\r\n        \r\n        for optic_object in self.__optics_objects:\r\n            if optic_object.processed is False:\r\n                self.__expand_cluster_order(optic_object)\r\n        \r\n        self.__extract_clusters()", "response": "Initializes the cluster allocation and builds the cluster ordering diagram."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef get_ordering(self):\r\n        \r\n        if self.__ordering is None:\r\n            self.__ordering = []\r\n        \r\n            for cluster in self.__clusters:\r\n                for index_object in cluster:\r\n                    optics_object = self.__optics_objects[index_object]\r\n                    if optics_object.reachability_distance is not None:\r\n                        self.__ordering.append(optics_object.reachability_distance)\r\n            \r\n        return self.__ordering", "response": "Returns the internal clustering ordering of the input data - set."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncreate a neighbor searcher in line with data type.", "response": "def __create_neighbor_searcher(self, data_type):\r\n        \"\"\"!\r\n        @brief Returns neighbor searcher in line with data type.\r\n\r\n        @param[in] data_type (string): Data type (points or distance matrix).\r\n\r\n        \"\"\"\r\n        if data_type == 'points':\r\n            return self.__neighbor_indexes_points\r\n        elif data_type == 'distance_matrix':\r\n            return self.__neighbor_indexes_distance_matrix\r\n        else:\r\n            raise TypeError(\"Unknown type of data is specified '%s'\" % data_type)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __expand_cluster_order(self, optics_object):\r\n        \r\n        optics_object.processed = True\r\n        \r\n        neighbors_descriptor = self.__neighbor_searcher(optics_object)\r\n        optics_object.reachability_distance = None\r\n        \r\n        self.__ordered_database.append(optics_object)\r\n        \r\n        # Check core distance\r\n        if len(neighbors_descriptor) >= self.__minpts:\r\n            neighbors_descriptor.sort(key = lambda obj: obj[1])\r\n            optics_object.core_distance = neighbors_descriptor[self.__minpts - 1][1]\r\n            \r\n            # Continue processing\r\n            order_seed = list()\r\n            self.__update_order_seed(optics_object, neighbors_descriptor, order_seed)\r\n            \r\n            while len(order_seed) > 0:\r\n                optic_descriptor = order_seed[0]\r\n                order_seed.remove(optic_descriptor)\r\n                \r\n                neighbors_descriptor = self.__neighbor_searcher(optic_descriptor)\r\n                optic_descriptor.processed = True\r\n                \r\n                self.__ordered_database.append(optic_descriptor)\r\n                \r\n                if len(neighbors_descriptor) >= self.__minpts:\r\n                    neighbors_descriptor.sort(key = lambda obj: obj[1])\r\n                    optic_descriptor.core_distance = neighbors_descriptor[self.__minpts - 1][1]\r\n                    \r\n                    self.__update_order_seed(optic_descriptor, neighbors_descriptor, order_seed)\r\n                else:\r\n                    optic_descriptor.core_distance = None\r\n                    \r\n        else:\r\n            optics_object.core_distance = None", "response": "Expand the cluster order from not processed optics - object that corresponds to object from input data."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nextract clusters and noise from ordered database.", "response": "def __extract_clusters(self):\r\n        \"\"\"!\r\n        @brief Extract clusters and noise from order database.\r\n        \r\n        \"\"\"\r\n     \r\n        self.__clusters = []\r\n        self.__noise = []\r\n\r\n        current_cluster = self.__noise\r\n        for optics_object in self.__ordered_database:\r\n            if (optics_object.reachability_distance is None) or (optics_object.reachability_distance > self.__eps):\r\n                if (optics_object.core_distance is not None) and (optics_object.core_distance <= self.__eps):\r\n                    self.__clusters.append([ optics_object.index_object ])\r\n                    current_cluster = self.__clusters[-1]\r\n                else:\r\n                    self.__noise.append(optics_object.index_object)\r\n            else:\r\n                current_cluster.append(optics_object.index_object)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\nupdate the list of reachable objects in the order of the given optics - object.", "response": "def __update_order_seed(self, optic_descriptor, neighbors_descriptors, order_seed):\r\n        \"\"\"!\r\n        @brief Update sorted list of reachable objects (from core-object) that should be processed using neighbors of core-object.\r\n        \r\n        @param[in] optic_descriptor (optics_descriptor): Core-object whose neighbors should be analysed.\r\n        @param[in] neighbors_descriptors (list): List of neighbors of core-object.\r\n        @param[in|out] order_seed (list): List of sorted object in line with reachable distance.\r\n        \r\n        \"\"\"\r\n        \r\n        for neighbor_descriptor in neighbors_descriptors:\r\n            index_neighbor = neighbor_descriptor[0]\r\n            current_reachable_distance = neighbor_descriptor[1]\r\n            \r\n            if self.__optics_objects[index_neighbor].processed is not True:\r\n                reachable_distance = max(current_reachable_distance, optic_descriptor.core_distance)\r\n                if self.__optics_objects[index_neighbor].reachability_distance is None:\r\n                    self.__optics_objects[index_neighbor].reachability_distance = reachable_distance\r\n                    \r\n                    # insert element in queue O(n) - worst case.\r\n                    index_insertion = len(order_seed)\r\n                    for index_seed in range(0, len(order_seed)):\r\n                        if reachable_distance < order_seed[index_seed].reachability_distance:\r\n                            index_insertion = index_seed\r\n                            break\r\n                    \r\n                    order_seed.insert(index_insertion, self.__optics_objects[index_neighbor])\r\n\r\n                else:\r\n                    if reachable_distance < self.__optics_objects[index_neighbor].reachability_distance:\r\n                        self.__optics_objects[index_neighbor].reachability_distance = reachable_distance\r\n                        order_seed.sort(key = lambda obj: obj.reachability_distance)"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __neighbor_indexes_points(self, optic_object):\r\n        kdnodes = self.__kdtree.find_nearest_dist_nodes(self.__sample_pointer[optic_object.index_object], self.__eps)\r\n        return [[node_tuple[1].payload, math.sqrt(node_tuple[0])] for node_tuple in kdnodes if\r\n                node_tuple[1].payload != optic_object.index_object]", "response": "Returns neighbors of the specified object in case of sequence of points."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn neighbors of the specified object in case of distance matrix.", "response": "def __neighbor_indexes_distance_matrix(self, optic_object):\r\n        \"\"\"!\r\n        @brief Return neighbors of the specified object in case of distance matrix.\r\n\r\n        @param[in] optic_object (optics_descriptor): Object for which neighbors should be returned in line with connectivity radius.\r\n\r\n        @return (list) List of indexes of neighbors in line the connectivity radius.\r\n\r\n        \"\"\"\r\n        distances = self.__sample_pointer[optic_object.index_object]\r\n        return [[index_neighbor, distances[index_neighbor]] for index_neighbor in range(len(distances))\r\n                if ((distances[index_neighbor] <= self.__eps) and (index_neighbor != optic_object.index_object))]"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __extract_features(self):\r\n        \r\n        self.__features = [];\r\n        \r\n        if (len(self.__tree.leafes) == 1):\r\n            # parameters are too general, copy all entries\r\n            for entry in self.__tree.leafes[0].entries:\r\n                self.__features.append(entry);\r\n\r\n        else:\r\n            # copy all leaf clustering features\r\n            for node in self.__tree.leafes:\r\n                self.__features.append(node.feature);", "response": "Extract features from CF - tree cluster."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ndecoding the data from the CF - tree features.", "response": "def __decode_data(self):\r\n        \"\"\"!\r\n        @brief Decodes data from CF-tree features.\r\n        \r\n        \"\"\"\r\n        \r\n        self.__clusters = [ [] for _ in range(self.__number_clusters) ];\r\n        self.__noise = [];\r\n        \r\n        for index_point in range(0, len(self.__pointer_data)):\r\n            (_, cluster_index) = self.__get_nearest_feature(self.__pointer_data[index_point], self.__features);\r\n            \r\n            self.__clusters[cluster_index].append(index_point);"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef __insert_data(self):\r\n        \r\n        for index_point in range(0, len(self.__pointer_data)):\r\n            point = self.__pointer_data[index_point];\r\n            self.__tree.insert_cluster( [ point ] );\r\n            \r\n            if (self.__tree.amount_entries > self.__entry_size_limit):\r\n                self.__tree = self.__rebuild_tree(index_point);", "response": "Inserts input data into the tree."}
{"SOURCE": "codesearchnet", "instruction": "Can you create a Python 3 function that\nrebuilds the tree with encoded points till specified point from input data space.", "response": "def __rebuild_tree(self, index_point):\r\n        \"\"\"!\r\n        @brief Rebuilt tree in case of maxumum number of entries is exceeded.\r\n        \r\n        @param[in] index_point (uint): Index of point that is used as end point of re-building.\r\n        \r\n        @return (cftree) Rebuilt tree with encoded points till specified point from input data space.\r\n        \r\n        \"\"\"\r\n        \r\n        rebuild_result = False;\r\n        increased_diameter = self.__tree.threshold * self.__diameter_multiplier;\r\n        \r\n        tree = None;\r\n        \r\n        while(rebuild_result is False):\r\n            # increase diameter and rebuild tree\r\n            if (increased_diameter == 0.0):\r\n                increased_diameter = 1.0;\r\n            \r\n            # build tree with update parameters\r\n            tree = cftree(self.__tree.branch_factor, self.__tree.max_entries, increased_diameter, self.__tree.type_measurement);\r\n            \r\n            for index_point in range(0, index_point + 1):\r\n                point = self.__pointer_data[index_point];\r\n                tree.insert_cluster([point]);\r\n            \r\n                if (tree.amount_entries > self.__entry_size_limit):\r\n                    increased_diameter *= self.__diameter_multiplier;\r\n                    continue;\r\n            \r\n            # Re-build is successful.\r\n            rebuild_result = True;\r\n        \r\n        return tree;"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __find_nearest_cluster_features(self):\r\n        \r\n        minimum_distance = float(\"Inf\");\r\n        index1 = 0;\r\n        index2 = 0;\r\n        \r\n        for index_candidate1 in range(0, len(self.__features)):\r\n            feature1 = self.__features[index_candidate1];\r\n            for index_candidate2 in range(index_candidate1 + 1, len(self.__features)):\r\n                feature2 = self.__features[index_candidate2];\r\n                \r\n                distance = feature1.get_distance(feature2, self.__measurement_type);\r\n                if (distance < minimum_distance):\r\n                    minimum_distance = distance;\r\n                    \r\n                    index1 = index_candidate1;\r\n                    index2 = index_candidate2;\r\n        \r\n        return [index1, index2];", "response": "Find pair of nearest CF entries."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef __get_nearest_feature(self, point, feature_collection):\r\n        \r\n        minimum_distance = float(\"Inf\");\r\n        index_nearest_feature = -1;\r\n        \r\n        for index_entry in range(0, len(feature_collection)):\r\n            point_entry = cfentry(1, linear_sum([ point ]), square_sum([ point ]));\r\n            \r\n            distance = feature_collection[index_entry].get_distance(point_entry, self.__measurement_type);\r\n            if (distance < minimum_distance):\r\n                minimum_distance = distance;\r\n                index_nearest_feature = index_entry;\r\n                \r\n        return (minimum_distance, index_nearest_feature);", "response": "Returns the nearest feature for the specified point."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __read_answer_from_line(self, index_point, line):\r\n\r\n        if line[0] == 'n':\r\n            self.__noise.append(index_point)\r\n        else:\r\n            index_cluster = int(line)\r\n            if index_cluster >= len(self.__clusters):\r\n                self.__clusters.append([index_point])\r\n            else:\r\n                self.__clusters[index_cluster].append(index_point)", "response": "Read information about point from a specific line and place it to the appropriate cluster or noise list."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef __read_answer(self):\r\n\r\n        if self.__clusters is not None:\r\n            return\r\n\r\n        file = open(self.__answer_path, 'r')\r\n\r\n        self.__clusters, self.__noise = [], []\r\n\r\n        index_point = 0\r\n        for line in file:\r\n            self.__read_answer_from_line(index_point, line)\r\n            index_point += 1\r\n\r\n        file.close()", "response": "Read information about proper clusters and noises from the file."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef append_cluster(self, cluster, data = None, marker = '.', markersize = None, color = None):\r\n        if len(cluster) == 0:\r\n            raise ValueError(\"Empty cluster is provided.\")\r\n\r\n        markersize = markersize or 5\r\n        if color is None:\r\n            index_color = len(self.__clusters) % len(color_list.TITLES)\r\n            color = color_list.TITLES[index_color]\r\n\r\n        cluster_descriptor = canvas_cluster_descr(cluster, data, marker, markersize, color)\r\n        self.__clusters.append(cluster_descriptor)", "response": "Append a cluster to the list of clusters."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef append_clusters(self, clusters, data=None, marker='.', markersize=None):\r\n\r\n        for cluster in clusters:\r\n            self.append_cluster(cluster, data, marker, markersize)", "response": "Append a list of clusters to the end of the list."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef show(self, pair_filter=None, **kwargs):\r\n\r\n        if not len(self.__clusters) > 0:\r\n            raise ValueError(\"There is no non-empty clusters for visualization.\")\r\n\r\n        cluster_data = self.__clusters[0].data or self.__clusters[0].cluster\r\n        dimension = len(cluster_data[0])\r\n\r\n        acceptable_pairs = pair_filter or []\r\n        pairs = []\r\n        amount_axis = 1\r\n        axis_storage = []\r\n\r\n        if dimension > 1:\r\n            pairs = self.__create_pairs(dimension, acceptable_pairs)\r\n            amount_axis = len(pairs)\r\n\r\n        self.__figure = plt.figure()\r\n        self.__grid_spec = self.__create_grid_spec(amount_axis, kwargs.get('max_row_size', 4))\r\n\r\n        for index in range(amount_axis):\r\n            ax = self.__create_canvas(dimension, pairs, index, **kwargs)\r\n            axis_storage.append(ax)\r\n\r\n        for cluster_descr in self.__clusters:\r\n            self.__draw_canvas_cluster(axis_storage, cluster_descr, pairs)\r\n\r\n        plt.show()", "response": "Displays clusters in multi - dimensional space."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef __create_grid_spec(self, amount_axis, max_row_size):\r\n        row_size = amount_axis\r\n        if row_size > max_row_size:\r\n            row_size = max_row_size\r\n\r\n        col_size = math.ceil(amount_axis / row_size)\r\n        return gridspec.GridSpec(col_size, row_size)", "response": "Create a grid specification for the given amount of canvases on a given axis."}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __create_pairs(self, dimension, acceptable_pairs):\r\n        if len(acceptable_pairs) > 0:\r\n            return acceptable_pairs\r\n\r\n        return list(itertools.combinations(range(dimension), 2))", "response": "Create coordinate pairs that should be displayed."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __create_canvas(self, dimension, pairs, position, **kwargs):\r\n        visible_grid = kwargs.get('visible_grid', True)\r\n        visible_labels = kwargs.get('visible_labels', True)\r\n        visible_axis = kwargs.get('visible_axis', False)\r\n\r\n        ax = self.__figure.add_subplot(self.__grid_spec[position])\r\n\r\n        if dimension > 1:\r\n            if visible_labels:\r\n                ax.set_xlabel(\"x%d\" % pairs[position][0])\r\n                ax.set_ylabel(\"x%d\" % pairs[position][1])\r\n        else:\r\n            ax.set_ylim(-0.5, 0.5)\r\n            ax.set_yticklabels([])\r\n\r\n        if visible_grid:\r\n            ax.grid(True)\r\n\r\n        if not visible_axis:\r\n            ax.set_yticklabels([])\r\n            ax.set_xticklabels([])\r\n\r\n        return ax", "response": "Create a new canvas with user defined parameters to display cluster or chunk of cluster on it."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef __draw_canvas_cluster(self, axis_storage, cluster_descr, pairs):\r\n\r\n        for index_axis in range(len(axis_storage)):\r\n            for item in cluster_descr.cluster:\r\n                if len(pairs) > 0:\r\n                    self.__draw_cluster_item_multi_dimension(axis_storage[index_axis], pairs[index_axis], item, cluster_descr)\r\n                else:\r\n                    self.__draw_cluster_item_one_dimension(axis_storage[index_axis], item, cluster_descr)", "response": "Draw a single or multiple cluster in a canvas."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ndrawing a single cluster chunk defined by pair coordinates in data space with dimension greater than 1.", "response": "def __draw_cluster_item_multi_dimension(self, ax, pair, item, cluster_descr):\r\n        \"\"\"!\r\n        @brief Draw cluster chunk defined by pair coordinates in data space with dimension greater than 1.\r\n\r\n        @param[in] ax (axis): Matplotlib axis that is used to display chunk of cluster point.\r\n        @param[in] pair (list): Coordinate of the point that should be displayed.\r\n        @param[in] item (list): Data point or index of data point.\r\n        @param[in] cluster_descr (canvas_cluster_descr): Cluster description whose point is visualized.\r\n\r\n        \"\"\"\r\n\r\n        index_dimension1 = pair[0]\r\n        index_dimension2 = pair[1]\r\n\r\n        if cluster_descr.data is None:\r\n            ax.plot(item[index_dimension1], item[index_dimension2],\r\n                    color=cluster_descr.color, marker=cluster_descr.marker, markersize=cluster_descr.markersize)\r\n        else:\r\n            ax.plot(cluster_descr.data[item][index_dimension1], cluster_descr.data[item][index_dimension2],\r\n                    color=cluster_descr.color, marker=cluster_descr.marker, markersize=cluster_descr.markersize)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef __draw_cluster_item_one_dimension(self, ax, item, cluster_descr):\r\n\r\n        if cluster_descr.data is None:\r\n            ax.plot(item[0], 0.0,\r\n                    color=cluster_descr.color, marker=cluster_descr.marker, markersize=cluster_descr.markersize)\r\n        else:\r\n            ax.plot(cluster_descr.data[item][0], 0.0,\r\n                    color=cluster_descr.color, marker=cluster_descr.marker, markersize=cluster_descr.markersize)", "response": "Draws one dimensional cluster point in one dimensional data space."}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\nadds a new cluster to the canvas for drawing.", "response": "def append_cluster(self, cluster, data=None, canvas=0, marker='.', markersize=None, color=None):\r\n        \"\"\"!\r\n        @brief Appends cluster to canvas for drawing.\r\n        \r\n        @param[in] cluster (list): cluster that may consist of indexes of objects from the data or object itself.\r\n        @param[in] data (list): If defines that each element of cluster is considered as a index of object from the data.\r\n        @param[in] canvas (uint): Number of canvas that should be used for displaying cluster.\r\n        @param[in] marker (string): Marker that is used for displaying objects from cluster on the canvas.\r\n        @param[in] markersize (uint): Size of marker.\r\n        @param[in] color (string): Color of marker.\r\n        \r\n        @return Returns index of cluster descriptor on the canvas.\r\n        \r\n        \"\"\"\r\n        \r\n        if len(cluster) == 0:\r\n            return\r\n        \r\n        if canvas > self.__number_canvases or canvas < 0:\r\n            raise ValueError(\"Canvas index '%d' is out of range [0; %d].\" % self.__number_canvases or canvas)\r\n        \r\n        if color is None:\r\n            index_color = len(self.__canvas_clusters[canvas]) % len(color_list.TITLES)\r\n            color = color_list.TITLES[index_color]\r\n        \r\n        added_canvas_descriptor = canvas_cluster_descr(cluster, data, marker, markersize, color)\r\n        self.__canvas_clusters[canvas].append( added_canvas_descriptor )\r\n\r\n        if data is None:\r\n            dimension = len(cluster[0])\r\n            if self.__canvas_dimensions[canvas] is None:\r\n                self.__canvas_dimensions[canvas] = dimension\r\n            elif self.__canvas_dimensions[canvas] != dimension:\r\n                raise ValueError(\"Only clusters with the same dimension of objects can be displayed on canvas.\")\r\n                \r\n        else:\r\n            dimension = len(data[0])\r\n            if self.__canvas_dimensions[canvas] is None:\r\n                self.__canvas_dimensions[canvas] = dimension\r\n            elif self.__canvas_dimensions[canvas] != dimension:\r\n                raise ValueError(\"Only clusters with the same dimension of objects can be displayed on canvas.\")\r\n\r\n        if (dimension < 1) or (dimension > 3):\r\n            raise ValueError(\"Only objects with size dimension 1 (1D plot), 2 (2D plot) or 3 (3D plot) \"\r\n                             \"can be displayed. For multi-dimensional data use 'cluster_visualizer_multidim'.\")\r\n        \r\n        if markersize is None:\r\n            if (dimension == 1) or (dimension == 2):\r\n                added_canvas_descriptor.markersize = self.__default_2d_marker_size\r\n            elif dimension == 3:\r\n                added_canvas_descriptor.markersize = self.__default_3d_marker_size\r\n        \r\n        return len(self.__canvas_clusters[canvas]) - 1"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef append_cluster_attribute(self, index_canvas, index_cluster, data, marker = None, markersize = None):\r\n        \r\n        cluster_descr = self.__canvas_clusters[index_canvas][index_cluster]\r\n        attribute_marker = marker\r\n        if attribute_marker is None:\r\n            attribute_marker = cluster_descr.marker\r\n        \r\n        attribure_markersize = markersize\r\n        if attribure_markersize is None:\r\n            attribure_markersize = cluster_descr.markersize\r\n        \r\n        attribute_color = cluster_descr.color\r\n        \r\n        added_attribute_cluster_descriptor = canvas_cluster_descr(data, None, attribute_marker, attribure_markersize, attribute_color)\r\n        self.__canvas_clusters[index_canvas][index_cluster].attributes.append(added_attribute_cluster_descriptor)", "response": "Append a cluster attribute to the list of attributes that are visualized for specific cluster on specific canvas."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef set_canvas_title(self, text, canvas = 0):\r\n        \r\n        if canvas > self.__number_canvases:\r\n            raise NameError('Canvas does ' + canvas + ' not exists.')\r\n        \r\n        self.__canvas_titles[canvas] = text", "response": "Set the title for the specified canvas."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ndisplaying the custers in a figure.", "response": "def show(self, figure=None, invisible_axis=True, visible_grid=True, display=True, shift=None):\r\n        \"\"\"!\r\n        @brief Shows clusters (visualize).\r\n        \r\n        @param[in] figure (fig): Defines requirement to use specified figure, if None - new figure is created for drawing clusters.\r\n        @param[in] invisible_axis (bool): Defines visibility of axes on each canvas, if True - axes are invisible.\r\n        @param[in] visible_grid (bool): Defines visibility of grid on each canvas, if True - grid is displayed.\r\n        @param[in] display (bool): Defines requirement to display clusters on a stage, if True - clusters are displayed,\r\n                    if False - plt.show() should be called by user.\"\r\n        @param[in] shift (uint): Force canvas shift value - defines canvas index from which custers should be visualized.\r\n        \r\n        @return (fig) Figure where clusters are shown.\r\n        \r\n        \"\"\"\r\n\r\n        canvas_shift = shift\r\n        if canvas_shift is None:\r\n            if figure is not None:\r\n                canvas_shift = len(figure.get_axes())\r\n            else:\r\n                canvas_shift = 0\r\n            \r\n        if figure is not None:\r\n            cluster_figure = figure\r\n        else:\r\n            cluster_figure = plt.figure()\r\n        \r\n        maximum_cols = self.__size_row\r\n        maximum_rows = math.ceil( (self.__number_canvases + canvas_shift) / maximum_cols)\r\n        \r\n        grid_spec = gridspec.GridSpec(maximum_rows, maximum_cols)\r\n\r\n        for index_canvas in range(len(self.__canvas_clusters)):\r\n            canvas_data = self.__canvas_clusters[index_canvas]\r\n            if len(canvas_data) == 0:\r\n                continue\r\n        \r\n            dimension = self.__canvas_dimensions[index_canvas]\r\n            \r\n            #ax = axes[real_index];\r\n            if (dimension == 1) or (dimension == 2):\r\n                ax = cluster_figure.add_subplot(grid_spec[index_canvas + canvas_shift])\r\n            else:\r\n                ax = cluster_figure.add_subplot(grid_spec[index_canvas + canvas_shift], projection='3d')\r\n            \r\n            if len(canvas_data) == 0:\r\n                plt.setp(ax, visible=False)\r\n            \r\n            for cluster_descr in canvas_data:\r\n                self.__draw_canvas_cluster(ax, dimension, cluster_descr)\r\n                \r\n                for attribute_descr in cluster_descr.attributes:\r\n                    self.__draw_canvas_cluster(ax, dimension, attribute_descr)\r\n            \r\n            if invisible_axis is True:\r\n                ax.xaxis.set_ticklabels([])\r\n                ax.yaxis.set_ticklabels([])\r\n                \r\n                if (dimension == 3):\r\n                    ax.zaxis.set_ticklabels([])\r\n            \r\n            if self.__canvas_titles[index_canvas] is not None:\r\n                ax.set_title(self.__canvas_titles[index_canvas])\r\n            \r\n            ax.grid(visible_grid)\r\n        \r\n        if display is True:\r\n            plt.show()\r\n        \r\n        return cluster_figure"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __draw_canvas_cluster(self, ax, dimension, cluster_descr):\r\n\r\n        cluster = cluster_descr.cluster\r\n        data = cluster_descr.data\r\n        marker = cluster_descr.marker\r\n        markersize = cluster_descr.markersize\r\n        color = cluster_descr.color\r\n        \r\n        for item in cluster:\r\n            if dimension == 1:\r\n                if data is None:\r\n                    ax.plot(item[0], 0.0, color = color, marker = marker, markersize = markersize)\r\n                else:\r\n                    ax.plot(data[item][0], 0.0, color = color, marker = marker, markersize = markersize)\r\n\r\n            elif dimension == 2:\r\n                if data is None:\r\n                    ax.plot(item[0], item[1], color = color, marker = marker, markersize = markersize)\r\n                else:\r\n                    ax.plot(data[item][0], data[item][1], color = color, marker = marker, markersize = markersize)\r\n        \r\n            elif dimension == 3:\r\n                if data is None:\r\n                    ax.scatter(item[0], item[1], item[2], c = color, marker = marker, s = markersize)\r\n                else:\r\n                    ax.scatter(data[item][0], data[item][1], data[item][2], c = color, marker = marker, s = markersize)", "response": "Draws the specified canvas cluster descriptor."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef gaussian(data, mean, covariance):\r\n    dimension = float(len(data[0]))\r\n \r\n    if dimension != 1.0:\r\n        inv_variance = numpy.linalg.pinv(covariance)\r\n    else:\r\n        inv_variance = 1.0 / covariance\r\n    \r\n    divider = (pi * 2.0) ** (dimension / 2.0) * numpy.sqrt(numpy.linalg.norm(covariance))\r\n    if divider != 0.0:\r\n        right_const = 1.0 / divider\r\n    else:\r\n        right_const = float('inf')\r\n    \r\n    result = []\r\n    \r\n    for point in data:\r\n        mean_delta = point - mean\r\n        point_gaussian = right_const * numpy.exp( -0.5 * mean_delta.dot(inv_variance).dot(numpy.transpose(mean_delta)) )\r\n        result.append(point_gaussian)\r\n    \r\n    return result", "response": "Calculates the gaussian function for each point in data using the specified mean and variance or covariance matrix in case of multi - dimensional data."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\ninitializing the parameters for the EM algorithm.", "response": "def initialize(self, init_type = ema_init_type.KMEANS_INITIALIZATION):\r\n        \"\"\"!\r\n        @brief Calculates initial parameters for EM algorithm: means and covariances using\r\n                specified strategy.\r\n        \r\n        @param[in] init_type (ema_init_type): Strategy for initialization.\r\n        \r\n        @return (float|list, float|numpy.array) Initial means and variance (covariance matrix in case multi-dimensional data).\r\n        \r\n        \"\"\"\r\n        if init_type == ema_init_type.KMEANS_INITIALIZATION:\r\n            return self.__initialize_kmeans()\r\n        \r\n        elif init_type == ema_init_type.RANDOM_INITIALIZATION:\r\n            return self.__initialize_random()\r\n        \r\n        raise NameError(\"Unknown type of EM algorithm initialization is specified.\")"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef __calculate_initial_clusters(self, centers):\r\n        \r\n        clusters = [[] for _ in range(len(centers))]\r\n        for index_point in range(len(self.__sample)):\r\n            index_optim, dist_optim = -1, 0.0\r\n             \r\n            for index in range(len(centers)):\r\n                dist = euclidean_distance_square(self.__sample[index_point], centers[index])\r\n                 \r\n                if (dist < dist_optim) or (index is 0):\r\n                    index_optim, dist_optim = index, dist\r\n             \r\n            clusters[index_optim].append(index_point)\r\n        \r\n        return clusters", "response": "Calculates the Euclidean distance to each point from each cluster and returns a list of clusters."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef notify(self, means, covariances, clusters):\r\n        self.__means_evolution.append(means)\r\n        self.__covariances_evolution.append(covariances)\r\n        self.__clusters_evolution.append(clusters)", "response": "This method is used by the algorithm to notify observer about changes where the algorithm should provide new values."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef show_clusters(clusters, sample, covariances, means, figure = None, display = True):\r\n        \r\n        visualizer = cluster_visualizer()\r\n        visualizer.append_clusters(clusters, sample)\r\n        \r\n        if figure is None:\r\n            figure = visualizer.show(display = False)\r\n        else:\r\n            visualizer.show(figure = figure, display = False)\r\n        \r\n        if len(sample[0]) == 2:\r\n            ema_visualizer.__draw_ellipses(figure, visualizer, clusters, covariances, means)\r\n\r\n        if display is True:\r\n            plt.show()\r\n\r\n        return figure", "response": "Displays clusters in case of two - dimensional dataset draws their ellipses."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef animate_cluster_allocation(data, observer, animation_velocity = 75, movie_fps = 1, save_movie = None):\r\n        \r\n        figure = plt.figure()\r\n        \r\n        def init_frame():\r\n            return frame_generation(0)\r\n        \r\n        def frame_generation(index_iteration):\r\n            figure.clf()\r\n            \r\n            figure.suptitle(\"EM algorithm (iteration: \" + str(index_iteration) +\")\", fontsize = 18, fontweight = 'bold')\r\n            \r\n            clusters = observer.get_evolution_clusters()[index_iteration]\r\n            covariances = observer.get_evolution_covariances()[index_iteration]\r\n            means = observer.get_evolution_means()[index_iteration]\r\n            \r\n            ema_visualizer.show_clusters(clusters, data, covariances, means, figure, False)\r\n            figure.subplots_adjust(top = 0.85)\r\n            \r\n            return [ figure.gca() ]\r\n\r\n        iterations = len(observer)\r\n        cluster_animation = animation.FuncAnimation(figure, frame_generation, iterations, interval = animation_velocity, init_func = init_frame, repeat_delay = 5000)\r\n\r\n        if save_movie is not None:\r\n            cluster_animation.save(save_movie, writer = 'ffmpeg', fps = movie_fps, bitrate = 1500)\r\n        else:\r\n            plt.show()", "response": "Animate the cluster allocation of the EM algorithm."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncalculate the Euclidean distance between two arrays using numpy.", "response": "def euclidean_distance_numpy(object1, object2):\r\n    \"\"\"!\r\n    @brief Calculate Euclidean distance between two objects using numpy.\r\n\r\n    @param[in] object1 (array_like): The first array_like object.\r\n    @param[in] object2 (array_like): The second array_like object.\r\n\r\n    @return (double) Euclidean distance between two objects.\r\n\r\n    \"\"\"\r\n    return numpy.sum(numpy.sqrt(numpy.square(object1 - object2)), axis=1).T"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncalculates the Euclidean distance between two vectors.", "response": "def euclidean_distance_square(point1, point2):\r\n    \"\"\"!\r\n    @brief Calculate square Euclidean distance between two vectors.\r\n\r\n    \\f[\r\n    dist(a, b) = \\sum_{i=0}^{N}(a_{i} - b_{i})^{2};\r\n    \\f]\r\n\r\n    @param[in] point1 (array_like): The first vector.\r\n    @param[in] point2 (array_like): The second vector.\r\n\r\n    @return (double) Square Euclidean distance between two vectors.\r\n\r\n    @see euclidean_distance, manhattan_distance, chebyshev_distance\r\n\r\n    \"\"\"\r\n    distance = 0.0\r\n    for i in range(len(point1)):\r\n        distance += (point1[i] - point2[i]) ** 2.0\r\n\r\n    return distance"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncalculate the square Euclidean distance between two arrays using numpy.", "response": "def euclidean_distance_square_numpy(object1, object2):\r\n    \"\"\"!\r\n    @brief Calculate square Euclidean distance between two objects using numpy.\r\n\r\n    @param[in] object1 (array_like): The first array_like object.\r\n    @param[in] object2 (array_like): The second array_like object.\r\n\r\n    @return (double) Square Euclidean distance between two objects.\r\n\r\n    \"\"\"\r\n    return numpy.sum(numpy.square(object1 - object2), axis=1).T"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef manhattan_distance(point1, point2):\r\n    distance = 0.0\r\n    dimension = len(point1)\r\n\r\n    for i in range(dimension):\r\n        distance += abs(point1[i] - point2[i])\r\n\r\n    return distance", "response": "Calculates the Manhattan distance between two vectors."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncalculate the Manhattan distance between two arrays using numpy.", "response": "def manhattan_distance_numpy(object1, object2):\r\n    \"\"\"!\r\n    @brief Calculate Manhattan distance between two objects using numpy.\r\n\r\n    @param[in] object1 (array_like): The first array_like object.\r\n    @param[in] object2 (array_like): The second array_like object.\r\n\r\n    @return (double) Manhattan distance between two objects.\r\n\r\n    \"\"\"\r\n    return numpy.sum(numpy.absolute(object1 - object2), axis=1).T"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncalculate the Chebyshev distance between two vectors.", "response": "def chebyshev_distance(point1, point2):\r\n    \"\"\"!\r\n    @brief Calculate Chebyshev distance between between two vectors.\r\n\r\n    \\f[\r\n    dist(a, b) = \\max_{}i\\left (\\left | a_{i} - b_{i} \\right |\\right );\r\n    \\f]\r\n\r\n    @param[in] point1 (array_like): The first vector.\r\n    @param[in] point2 (array_like): The second vector.\r\n\r\n    @return (double) Chebyshev distance between two vectors.\r\n\r\n    @see euclidean_distance_square, euclidean_distance, minkowski_distance\r\n\r\n    \"\"\"\r\n    distance = 0.0\r\n    dimension = len(point1)\r\n\r\n    for i in range(dimension):\r\n        distance = max(distance, abs(point1[i] - point2[i]))\r\n\r\n    return distance"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef chebyshev_distance_numpy(object1, object2):\r\n    return numpy.max(numpy.absolute(object1 - object2), axis=1).T", "response": "Calculates the Chebyshev distance between two arrays using numpy."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef minkowski_distance(point1, point2, degree=2):\r\n    distance = 0.0\r\n    for i in range(len(point1)):\r\n        distance += (point1[i] - point2[i]) ** degree\r\n\r\n    return distance ** (1.0 / degree)", "response": "Calculates the Minkowski distance between two points."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalculates the Minkowski distance between two arrays using numpy.", "response": "def minkowski_distance_numpy(object1, object2, degree=2):\r\n    \"\"\"!\r\n    @brief Calculate Minkowski distance between objects using numpy.\r\n\r\n    @param[in] object1 (array_like): The first array_like object.\r\n    @param[in] object2 (array_like): The second array_like object.\r\n    @param[in] degree (numeric): Degree of that is used for Minkowski distance.\r\n\r\n    @return (double) Minkowski distance between two object.\r\n\r\n    \"\"\"\r\n    return numpy.sum(numpy.power(numpy.power(object1 - object2, degree), 1/degree), axis=1).T"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncalculate the Canberra distance between two objects using numpy.", "response": "def canberra_distance_numpy(object1, object2):\r\n    \"\"\"!\r\n    @brief Calculate Canberra distance between two objects using numpy.\r\n\r\n    @param[in] object1 (array_like): The first vector.\r\n    @param[in] object2 (array_like): The second vector.\r\n\r\n    @return (float) Canberra distance between two objects.\r\n\r\n    \"\"\"\r\n    with numpy.errstate(divide='ignore', invalid='ignore'):\r\n        result = numpy.divide(numpy.abs(object1 - object2), numpy.abs(object1) + numpy.abs(object2))\r\n\r\n    if len(result.shape) > 1:\r\n        return numpy.sum(numpy.nan_to_num(result), axis=1).T\r\n    else:\r\n        return numpy.sum(numpy.nan_to_num(result))"}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ncalculates the Chi square distance between two vectors.", "response": "def chi_square_distance(point1, point2):\r\n    \"\"\"!\r\n    @brief Calculate Chi square distance between two vectors.\r\n\r\n    \\f[\r\n    dist(a, b) = \\sum_{i=0}^{N}\\frac{\\left ( a_{i} - b_{i} \\right )^{2}}{\\left | a_{i} \\right | + \\left | b_{i} \\right |};\r\n    \\f]\r\n\r\n    @param[in] point1 (array_like): The first vector.\r\n    @param[in] point2 (array_like): The second vector.\r\n\r\n    @return (float) Chi square distance between two objects.\r\n\r\n    \"\"\"\r\n    distance = 0.0\r\n    for i in range(len(point1)):\r\n        divider = abs(point1[i]) + abs(point2[i])\r\n        if divider == 0.0:\r\n            continue\r\n\r\n        distance += ((point1[i] - point2[i]) ** 2.0) / divider\r\n\r\n    return distance"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nenables numpy usage for the current time - domain entry.", "response": "def enable_numpy_usage(self):\r\n        \"\"\"!\r\n        @brief Start numpy for distance calculation.\r\n        @details Useful in case matrices to increase performance. No effect in case of type_metric.USER_DEFINED type.\r\n\r\n        \"\"\"\r\n        self.__numpy = True\r\n        if self.__type != type_metric.USER_DEFINED:\r\n            self.__calculator = self.__create_distance_calculator()"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __create_distance_calculator_numpy(self):\r\n        if self.__type == type_metric.EUCLIDEAN:\r\n            return euclidean_distance_numpy\r\n\r\n        elif self.__type == type_metric.EUCLIDEAN_SQUARE:\r\n            return euclidean_distance_square_numpy\r\n\r\n        elif self.__type == type_metric.MANHATTAN:\r\n            return manhattan_distance_numpy\r\n\r\n        elif self.__type == type_metric.CHEBYSHEV:\r\n            return chebyshev_distance_numpy\r\n\r\n        elif self.__type == type_metric.MINKOWSKI:\r\n            return lambda object1, object2: minkowski_distance_numpy(object1, object2, self.__args.get('degree', 2))\r\n\r\n        elif self.__type == type_metric.CANBERRA:\r\n            return canberra_distance_numpy\r\n\r\n        elif self.__type == type_metric.CHI_SQUARE:\r\n            return chi_square_distance_numpy\r\n\r\n        elif self.__type == type_metric.USER_DEFINED:\r\n            return self.__func\r\n\r\n        else:\r\n            raise ValueError(\"Unknown type of metric: '%d'\", self.__type)", "response": "Creates a function that returns the distance metric calculator that uses numpy."}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\nreturns the number of oscillations in the specified time series.", "response": "def extract_number_oscillations(self, index, amplitude_threshold):\r\n        \"\"\"!\r\n        @brief Extracts number of oscillations of specified oscillator.\r\n        \r\n        @param[in] index (uint): Index of oscillator whose dynamic is considered.\r\n        @param[in] amplitude_threshold (double): Amplitude threshold when oscillation is taken into account, for example,\r\n                    when oscillator amplitude is greater than threshold then oscillation is incremented.\r\n        \r\n        @return (uint) Number of oscillations of specified oscillator.\r\n        \r\n        \"\"\"\r\n        \r\n        return pyclustering.utils.extract_number_oscillations(self.__amplitude, index, amplitude_threshold);"}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef show_output_dynamic(fsync_output_dynamic):\r\n        \r\n        pyclustering.utils.draw_dynamics(fsync_output_dynamic.time, fsync_output_dynamic.output, x_title = \"t\", y_title = \"amplitude\");", "response": "Show the dynamic output of each oscillator during simulation."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nsimulating the dynamic of the object.", "response": "def simulate(self, steps, time, collect_dynamic = False):\r\n        \"\"\"!\r\n        @brief Performs static simulation of oscillatory network.\r\n        \r\n        @param[in] steps (uint): Number simulation steps.\r\n        @param[in] time (double): Time of simulation.\r\n        @param[in] collect_dynamic (bool): If True - returns whole dynamic of oscillatory network, otherwise returns only last values of dynamics.\r\n        \r\n        @return (list) Dynamic of oscillatory network. If argument 'collect_dynamic' is True, than return dynamic for the whole simulation time,\r\n                 otherwise returns only last values (last step of simulation) of output dynamic.\r\n        \r\n        @see simulate()\r\n        @see simulate_dynamic()\r\n        \r\n        \"\"\"\r\n        \r\n        dynamic_amplitude, dynamic_time = ([], []) if collect_dynamic is False else ([self.__amplitude], [0]);\r\n        \r\n        step = time / steps;\r\n        int_step = step / 10.0;\r\n        \r\n        for t in numpy.arange(step, time + step, step):\r\n            self.__amplitude = self.__calculate(t, step, int_step);\r\n            \r\n            if collect_dynamic is True:\r\n                dynamic_amplitude.append([ numpy.real(amplitude)[0] for amplitude in self.__amplitude ]);\r\n                dynamic_time.append(t);\r\n        \r\n        if collect_dynamic is False:\r\n            dynamic_amplitude.append([ numpy.real(amplitude)[0] for amplitude in self.__amplitude ]);\r\n            dynamic_time.append(time);\r\n\r\n        output_sync_dynamic = fsync_dynamic(dynamic_amplitude, dynamic_time);\r\n        return output_sync_dynamic;"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\ncalculate the new amplitudes for the given time step and returns a list of new states.", "response": "def __calculate(self, t, step, int_step):\r\n        \"\"\"!\r\n        @brief Calculates new amplitudes for oscillators in the network in line with current step.\r\n        \r\n        @param[in] t (double): Time of simulation.\r\n        @param[in] step (double): Step of solution at the end of which states of oscillators should be calculated.\r\n        @param[in] int_step (double): Step differentiation that is used for solving differential equation.\r\n        \r\n        @return (list) New states (phases) for oscillators.\r\n        \r\n        \"\"\"\r\n        \r\n        next_amplitudes = [0.0] * self._num_osc;\r\n        \r\n        for index in range (0, self._num_osc, 1):\r\n            z = numpy.array(self.__amplitude[index], dtype = numpy.complex128, ndmin = 1);\r\n            result = odeint(self.__calculate_amplitude, z.view(numpy.float64), numpy.arange(t - step, t, int_step), (index , ));\r\n            next_amplitudes[index] = (result[len(result) - 1]).view(numpy.complex128);\r\n        \r\n        return next_amplitudes;"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\ncalculate the constant oscillator property that is based on frequency and radius.", "response": "def __oscillator_property(self, index):\r\n        \"\"\"!\r\n        @brief Calculate Landau-Stuart oscillator constant property that is based on frequency and radius.\r\n        \r\n        @param[in] index (uint): Oscillator index whose property is calculated.\r\n        \r\n        @return (double) Oscillator property.\r\n        \r\n        \"\"\"\r\n        \r\n        return numpy.array(1j * self.__frequency[index] + self.__radius[index]**2, dtype = numpy.complex128, ndmin = 1);"}
{"SOURCE": "codesearchnet", "instruction": "How would you implement a function in Python 3 that\ncalculates the Landau - Stuart state of the specified oscillator.", "response": "def __landau_stuart(self, amplitude, index):\r\n        \"\"\"!\r\n        @brief Calculate Landau-Stuart state.\r\n        \r\n        @param[in] amplitude (double): Current amplitude of oscillator.\r\n        @param[in] index (uint): Oscillator index whose state is calculated. \r\n        \r\n        @return (double) Landau-Stuart state.\r\n        \r\n        \"\"\"\r\n        \r\n        return (self.__properties[index] - numpy.absolute(amplitude) ** 2) * amplitude;"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef __synchronization_mechanism(self, amplitude, index):\r\n        \r\n        sync_influence = 0.0;\r\n        \r\n        for k in range(self._num_osc):\r\n            if self.has_connection(index, k) is True:\r\n                amplitude_neighbor = numpy.array(self.__amplitude[k], dtype = numpy.complex128, ndmin = 1);\r\n                sync_influence += amplitude_neighbor - amplitude;\r\n        \r\n        return sync_influence * self.__coupling_strength / self._num_osc;", "response": "Calculates the synchronization part using Kuramoto synchronization mechanism."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __calculate_amplitude(self, amplitude, t, argv):\r\n        \r\n        z = amplitude.view(numpy.complex);\r\n        dzdt = self.__landau_stuart(z, argv) + self.__synchronization_mechanism(z, argv);\r\n        \r\n        return dzdt.view(numpy.float64);", "response": "Calculates the new amplitude value for particular oscillator that is defined by index that is in argv."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef small_mind_image_recognition():\r\n    images = [];\r\n    images += IMAGE_SYMBOL_SAMPLES.LIST_IMAGES_SYMBOL_M;\r\n    images += IMAGE_SYMBOL_SAMPLES.LIST_IMAGES_SYMBOL_I;\r\n    images += IMAGE_SYMBOL_SAMPLES.LIST_IMAGES_SYMBOL_N;\r\n    images += IMAGE_SYMBOL_SAMPLES.LIST_IMAGES_SYMBOL_D;\r\n    \r\n    template_recognition_image(images, 100, 10, 0.2);", "response": "Trains network using letters M I N D and recognize each of them with and without noise."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef small_abc_image_recognition():\r\n    images = [];\r\n    images += IMAGE_SYMBOL_SAMPLES.LIST_IMAGES_SYMBOL_A;\r\n    images += IMAGE_SYMBOL_SAMPLES.LIST_IMAGES_SYMBOL_B;\r\n    images += IMAGE_SYMBOL_SAMPLES.LIST_IMAGES_SYMBOL_C;\r\n    \r\n    template_recognition_image(images, 250, 25);", "response": "Trains network using letters A B C and recognize each of them with and without noise."}
{"SOURCE": "codesearchnet", "instruction": "Here you have a function in Python 3, explain what it does\ndef small_ftk_image_recognition():\r\n    images = [];\r\n    images += IMAGE_SYMBOL_SAMPLES.LIST_IMAGES_SYMBOL_F;\r\n    images += IMAGE_SYMBOL_SAMPLES.LIST_IMAGES_SYMBOL_T;\r\n    images += IMAGE_SYMBOL_SAMPLES.LIST_IMAGES_SYMBOL_K;\r\n    \r\n    template_recognition_image(images, 100, 10, 0.2);", "response": "Trains network using letters F T K and recognize each of them with and without noise."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nconvert chromosome to cluster representation", "response": "def get_clusters_representation(chromosome, count_clusters=None):\n        \"\"\" Convert chromosome to cluster representation:\n                chromosome : [0, 1, 1, 0, 2, 3, 3]\n                clusters: [[0, 3], [1, 2], [4], [5, 6]]\n        \"\"\"\n\n        if count_clusters is None:\n            count_clusters = ga_math.calc_count_centers(chromosome)\n\n        # Initialize empty clusters\n        clusters = [[] for _ in range(count_clusters)]\n\n        # Fill clusters with index of data\n        for _idx_data in range(len(chromosome)):\n            clusters[chromosome[_idx_data]].append(_idx_data)\n\n        return clusters"}
{"SOURCE": "codesearchnet", "instruction": "Can you write a function in Python 3 where it\ncalculates centres of chromosomes.", "response": "def get_centres(chromosomes, data, count_clusters):\n        \"\"\"!\n        \"\"\"\n\n        centres = ga_math.calc_centers(chromosomes, data, count_clusters)\n\n        return centres"}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\ncalculate the centers of the data in the cluster.", "response": "def calc_centers(chromosomes, data, count_clusters=None):\n        \"\"\"!\n        \"\"\"\n\n        if count_clusters is None:\n            count_clusters = ga_math.calc_count_centers(chromosomes[0])\n\n        # Initialize center\n        centers = np.zeros(shape=(len(chromosomes), count_clusters, len(data[0])))\n\n        for _idx_chromosome in range(len(chromosomes)):\n\n            # Get count data in clusters\n            count_data_in_cluster = np.zeros(count_clusters)\n\n            # Next data point\n            for _idx in range(len(chromosomes[_idx_chromosome])):\n\n                cluster_num = chromosomes[_idx_chromosome][_idx]\n\n                centers[_idx_chromosome][cluster_num] += data[_idx]\n                count_data_in_cluster[cluster_num] += 1\n\n            for _idx_cluster in range(count_clusters):\n                if count_data_in_cluster[_idx_cluster] != 0:\n                    centers[_idx_chromosome][_idx_cluster] /= count_data_in_cluster[_idx_cluster]\n\n        return centers"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating the probability vector of a set of fitness functions.", "response": "def calc_probability_vector(fitness):\n        \"\"\"!\n        \"\"\"\n\n        if len(fitness) == 0:\n            raise AttributeError(\"Has no any fitness functions.\")\n\n        # Get 1/fitness function\n        inv_fitness = np.zeros(len(fitness))\n\n        #\n        for _idx in range(len(inv_fitness)):\n\n            if fitness[_idx] != 0.0:\n                inv_fitness[_idx] = 1.0 / fitness[_idx]\n            else:\n                inv_fitness[_idx] = 0.0\n\n        # Initialize vector\n        prob = np.zeros(len(fitness))\n\n        # Initialize first element\n        prob[0] = inv_fitness[0]\n\n        # Accumulate values in probability vector\n        for _idx in range(1, len(inv_fitness)):\n            prob[_idx] = prob[_idx - 1] + inv_fitness[_idx]\n\n        # Normalize\n        prob /= prob[-1]\n\n        ga_math.set_last_value_to_one(prob)\n\n        return prob"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef set_last_value_to_one(probabilities):\n\n        # Start from the last elem\n        back_idx = - 1\n\n        # All values equal to the last elem should be set to 1\n        last_val = probabilities[back_idx]\n\n        # for all elements or if a elem not equal to the last elem\n        for _ in range(-1, -len(probabilities) - 1):\n            if probabilities[back_idx] == last_val:\n                probabilities[back_idx] = 1\n            else:\n                break", "response": "Update the last same probabilities to one."}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef cluster_sample1():\r\n    \"Start with wrong number of clusters.\"\r\n    start_centers = [[3.7, 5.5]]\r\n    template_clustering(start_centers, SIMPLE_SAMPLES.SAMPLE_SIMPLE1, criterion = splitting_type.BAYESIAN_INFORMATION_CRITERION)\r\n    template_clustering(start_centers, SIMPLE_SAMPLES.SAMPLE_SIMPLE1, criterion = splitting_type.MINIMUM_NOISELESS_DESCRIPTION_LENGTH)", "response": "Start with wrong number of clusters."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cluster_sample2():\r\n    \"Start with wrong number of clusters.\"\r\n    start_centers = [[3.5, 4.8], [2.6, 2.5]]\r\n    template_clustering(start_centers, SIMPLE_SAMPLES.SAMPLE_SIMPLE2, criterion = splitting_type.BAYESIAN_INFORMATION_CRITERION)\r\n    template_clustering(start_centers, SIMPLE_SAMPLES.SAMPLE_SIMPLE2, criterion = splitting_type.MINIMUM_NOISELESS_DESCRIPTION_LENGTH)", "response": "Start with wrong number of clusters."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nstarting with wrong number of clusters.", "response": "def cluster_sample3():\r\n    \"Start with wrong number of clusters.\"\r\n    start_centers = [[0.2, 0.1], [4.0, 1.0]]\r\n    template_clustering(start_centers, SIMPLE_SAMPLES.SAMPLE_SIMPLE3, criterion = splitting_type.BAYESIAN_INFORMATION_CRITERION)\r\n    template_clustering(start_centers, SIMPLE_SAMPLES.SAMPLE_SIMPLE3, criterion = splitting_type.MINIMUM_NOISELESS_DESCRIPTION_LENGTH)"}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function for\nstarting with wrong number of clusters.", "response": "def cluster_sample5():\r\n    \"Start with wrong number of clusters.\"\r\n    start_centers = [[0.0, 1.0], [0.0, 0.0]]\r\n    template_clustering(start_centers, SIMPLE_SAMPLES.SAMPLE_SIMPLE5, criterion = splitting_type.BAYESIAN_INFORMATION_CRITERION)\r\n    template_clustering(start_centers, SIMPLE_SAMPLES.SAMPLE_SIMPLE5, criterion = splitting_type.MINIMUM_NOISELESS_DESCRIPTION_LENGTH)"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef cluster_elongate():\r\n    \"Not so applicable for this sample\"\r\n    start_centers = [[1.0, 4.5], [3.1, 2.7]]\r\n    template_clustering(start_centers, SIMPLE_SAMPLES.SAMPLE_ELONGATE, criterion = splitting_type.BAYESIAN_INFORMATION_CRITERION)\r\n    template_clustering(start_centers, SIMPLE_SAMPLES.SAMPLE_ELONGATE, criterion = splitting_type.MINIMUM_NOISELESS_DESCRIPTION_LENGTH)", "response": "Not so applicable for this sample"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef cluster_lsun():\r\n    \"Not so applicable for this sample\"\r\n    start_centers = [[1.0, 3.5], [2.0, 0.5], [3.0, 3.0]]\r\n    template_clustering(start_centers, FCPS_SAMPLES.SAMPLE_LSUN, criterion = splitting_type.BAYESIAN_INFORMATION_CRITERION)\r\n    template_clustering(start_centers, FCPS_SAMPLES.SAMPLE_LSUN, criterion = splitting_type.MINIMUM_NOISELESS_DESCRIPTION_LENGTH)", "response": "Not so applicable for this sample"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef cluster_target():\r\n    \"Not so applicable for this sample\"\r\n    start_centers = [[0.2, 0.2], [0.0, -2.0], [3.0, -3.0], [3.0, 3.0], [-3.0, 3.0], [-3.0, -3.0]]\r\n    template_clustering(start_centers, FCPS_SAMPLES.SAMPLE_TARGET, criterion = splitting_type.BAYESIAN_INFORMATION_CRITERION)\r\n    template_clustering(start_centers, FCPS_SAMPLES.SAMPLE_TARGET, criterion = splitting_type.MINIMUM_NOISELESS_DESCRIPTION_LENGTH)", "response": "Not so applicable for this sample"}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef cluster_two_diamonds():\r\n    \"Start with wrong number of clusters.\"\r\n    start_centers = [[0.8, 0.2]]\r\n    template_clustering(start_centers, FCPS_SAMPLES.SAMPLE_TWO_DIAMONDS, criterion = splitting_type.BAYESIAN_INFORMATION_CRITERION)\r\n    template_clustering(start_centers, FCPS_SAMPLES.SAMPLE_TWO_DIAMONDS, criterion = splitting_type.MINIMUM_NOISELESS_DESCRIPTION_LENGTH)", "response": "Start with wrong number of clusters."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nstarting with wrong number of clusters.", "response": "def cluster_hepta():\r\n    \"Start with wrong number of clusters.\"\r\n    start_centers = [[0.0, 0.0, 0.0], [3.0, 0.0, 0.0], [-2.0, 0.0, 0.0], [0.0, 3.0, 0.0], [0.0, -3.0, 0.0], [0.0, 0.0, 2.5]]\r\n    template_clustering(start_centers, FCPS_SAMPLES.SAMPLE_HEPTA, criterion = splitting_type.BAYESIAN_INFORMATION_CRITERION)\r\n    template_clustering(start_centers, FCPS_SAMPLES.SAMPLE_HEPTA, criterion = splitting_type.MINIMUM_NOISELESS_DESCRIPTION_LENGTH)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\ncalculating new connectivity radius.", "response": "def __calculate_radius(self, number_neighbors, radius):\r\n        \"\"\"!\r\n        @brief Calculate new connectivity radius.\r\n        \r\n        @param[in] number_neighbors (uint): Average amount of neighbors that should be connected by new radius.\r\n        @param[in] radius (double): Current connectivity radius.\r\n        \r\n        @return New connectivity radius.\r\n        \r\n        \"\"\"\r\n        \r\n        if (number_neighbors >= len(self._osc_loc)):\r\n            return radius * self.__increase_persent + radius;\r\n        \r\n        return average_neighbor_distance(self._osc_loc, number_neighbors);"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef __store_dynamic(self, dyn_phase, dyn_time, analyser, begin_state):\r\n        \r\n        if (begin_state is True):\r\n            dyn_time.append(0);\r\n            dyn_phase.append(analyser.output[0]);\r\n        \r\n        else:\r\n            dyn_phase.append(analyser.output[len(analyser.output) - 1]);\r\n            dyn_time.append(len(dyn_time));", "response": "Store the specified state of a specific dynamic of Sync network to hSync."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nchanging the type of clusters encoding to specified type.", "response": "def set_encoding(self, encoding):\n        \"\"\"!\n        @brief Change clusters encoding to specified type (index list, object list, labeling).\n        \n        @param[in] encoding (type_encoding): New type of clusters representation.\n        \n        \"\"\"\n        \n        if(encoding == self.__type_representation):\n            return;\n        \n        if (self.__type_representation == type_encoding.CLUSTER_INDEX_LABELING):\n            if (encoding == type_encoding.CLUSTER_INDEX_LIST_SEPARATION):\n                self.__clusters = self.__convert_label_to_index();\n            \n            else:\n                self.__clusters = self.__convert_label_to_object();\n        \n        elif (self.__type_representation == type_encoding.CLUSTER_INDEX_LIST_SEPARATION):\n            if (encoding == type_encoding.CLUSTER_INDEX_LABELING):\n                self.__clusters = self.__convert_index_to_label();\n            \n            else:\n                self.__clusters = self.__convert_index_to_object();\n        \n        else:\n            if (encoding == type_encoding.CLUSTER_INDEX_LABELING):\n                self.__clusters = self.__convert_object_to_label();\n            \n            else:\n                self.__clusters = self.__convert_object_to_index();\n        \n        self.__type_representation = encoding;"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef process(self):\r\n        \r\n        if self.__ccore is True:\r\n            (self.__clusters, self.__noise) = wrapper.dbscan(self.__pointer_data, self.__eps, self.__neighbors, self.__data_type)\r\n            \r\n        else:\r\n            if self.__data_type == 'points':\r\n                self.__kdtree = kdtree(self.__pointer_data, range(len(self.__pointer_data)))\r\n\r\n            for i in range(0, len(self.__pointer_data)):\r\n                if self.__visited[i] is False:\r\n                    cluster = self.__expand_cluster(i)\r\n                    if cluster is not None:\r\n                        self.__clusters.append(cluster)\r\n\r\n            for i in range(0, len(self.__pointer_data)):\r\n                if self.__belong[i] is False:\r\n                    self.__noise.append(i)", "response": "This method is called by the class constructor when the class is instantiated."}
{"SOURCE": "codesearchnet", "instruction": "How would you code a function in Python 3 to\nreturn a list of indexes that belong to the same cluster and list of points that are not noise.", "response": "def __expand_cluster(self, index_point):\r\n        \"\"\"!\r\n        @brief Expands cluster from specified point in the input data space.\r\n        \r\n        @param[in] index_point (list): Index of a point from the data.\r\n\r\n        @return (list) Return tuple of list of indexes that belong to the same cluster and list of points that are marked as noise: (cluster, noise), or None if nothing has been expanded.\r\n        \r\n        \"\"\"\r\n        \r\n        cluster = None\r\n        self.__visited[index_point] = True\r\n        neighbors = self.__neighbor_searcher(index_point)\r\n         \r\n        if len(neighbors) >= self.__neighbors:\r\n            cluster = [index_point]\r\n             \r\n            self.__belong[index_point] = True\r\n             \r\n            for i in neighbors:\r\n                if self.__visited[i] is False:\r\n                    self.__visited[i] = True\r\n\r\n                    next_neighbors = self.__neighbor_searcher(i)\r\n                     \r\n                    if len(next_neighbors) >= self.__neighbors:\r\n                        neighbors += [k for k in next_neighbors if ( (k in neighbors) == False) and k != index_point]\r\n                 \r\n                if self.__belong[i] is False:\r\n                    cluster.append(i)\r\n                    self.__belong[i] = True\r\n\r\n        return cluster"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreturning the neighbors of the specified object in case of sequence of points.", "response": "def __neighbor_indexes_points(self, index_point):\r\n        \"\"\"!\r\n        @brief Return neighbors of the specified object in case of sequence of points.\r\n\r\n        @param[in] index_point (uint): Index point whose neighbors are should be found.\r\n\r\n        @return (list) List of indexes of neighbors in line the connectivity radius.\r\n\r\n        \"\"\"\r\n        kdnodes = self.__kdtree.find_nearest_dist_nodes(self.__pointer_data[index_point], self.__eps)\r\n        return [node_tuple[1].payload for node_tuple in kdnodes if node_tuple[1].payload != index_point]"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __neighbor_indexes_distance_matrix(self, index_point):\r\n        distances = self.__pointer_data[index_point]\r\n        return [index_neighbor for index_neighbor in range(len(distances))\r\n                if ((distances[index_neighbor] <= self.__eps) and (index_neighbor != index_point))]", "response": "Returns neighbors of the specified object in case of distance matrix."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate a brief explanation for the following Python 3 code\ndef generate(self):\r\n        data_points = []\r\n\r\n        for index_cluster in range(self.__amount_clusters):\r\n            for _ in range(self.__cluster_sizes[index_cluster]):\r\n                point = self.__generate_point(index_cluster)\r\n                data_points.append(point)\r\n\r\n        return data_points", "response": "Generates data points in line with generator parameters."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef __generate_point(self, index_cluster):\r\n        return [ random.gauss(self.__cluster_centers[index_cluster][index_dimension],\r\n                              self.__cluster_width[index_cluster] / 2.0)\r\n                 for index_dimension in range(self.__dimension) ]", "response": "Generate a new point in line with normal distribution and cluster parameters."}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef __generate_cluster_centers(self, width):\r\n        centers = []\r\n        default_offset = max(width) * 4.0\r\n        for i in range(self.__amount_clusters):\r\n            center = [ random.gauss(i * default_offset, width[i] / 2.0) for _ in range(self.__dimension) ]\r\n            centers.append(center)\r\n\r\n        return centers", "response": "Generate centers in line with normal distribution."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function for\nreading a graph from a file in GRPR format.", "response": "def read_graph(filename):\r\n    \"\"\"!\r\n    @brief Read graph from file in GRPR format.\r\n    \r\n    @param[in] filename (string): Path to file with graph in GRPR format.\r\n    \r\n    @return (graph) Graph that is read from file.\r\n    \r\n    \"\"\"\r\n    \r\n    file = open(filename, 'r');\r\n    \r\n    comments = \"\";\r\n    space_descr = [];\r\n    data = [];\r\n    data_type = None;\r\n    \r\n    map_data_repr = dict();   # Used as a temporary buffer only when input graph is represented by edges.\r\n    \r\n    for line in file:\r\n        if (line[0] == 'c' or line[0] == 'p'): \r\n            comments += line[1:]; \r\n        \r\n        elif (line[0] == 'r'): \r\n            node_coordinates = [float(val) for val in line[1:].split()];\r\n            if (len(node_coordinates) != 2):\r\n                raise NameError('Invalid format of space description for node (only 2-dimension space is supported)');\r\n                \r\n            space_descr.append( [float(val) for val in line[1:].split()] );\r\n        \r\n        elif (line[0] == 'm'):\r\n            if ( (data_type is not None) and (data_type != 'm') ):\r\n                raise NameError('Invalid format of graph representation (only one type should be used)');\r\n \r\n            data_type = 'm';\r\n            data.append( [float(val) for val in line[1:].split()] );\r\n        \r\n        elif (line[0] == 'v'):\r\n            if ( (data_type is not None) and (data_type != 'v') ):\r\n                raise NameError('Invalid format of graph representation (only one type should be used)');\r\n            \r\n            data_type = 'v';\r\n            data.append( [float(val) for val in line[1:].split()] );\r\n            \r\n        elif (line[0] == 'e'):\r\n            if ( (data_type is not None) and (data_type != 'e') ):\r\n                raise NameError('Invalid format of graph representation (only one type should be used)');\r\n               \r\n            data_type = 'e';\r\n            vertices = [int(val) for val in line[1:].split()];\r\n            \r\n            if (vertices[0] not in map_data_repr):\r\n                map_data_repr[ vertices[0] ] = [ vertices[1] ];\r\n            else:\r\n                map_data_repr[ vertices[0] ].append(vertices[1])\r\n                \r\n            if (vertices[1] not in map_data_repr):\r\n                map_data_repr[ vertices[1] ] = [ vertices[0] ];\r\n            else:\r\n                map_data_repr[ vertices[1] ].append(vertices[0]);\r\n            \r\n            \r\n        elif (len(line.strip()) == 0): continue;\r\n        \r\n        else: \r\n            print(line);\r\n            raise NameError('Invalid format of file with graph description');\r\n    \r\n    # In case of edge representation result should be copied.\r\n    if (data_type == 'e'):\r\n        for index in range(len(map_data_repr)):\r\n            data.append([0] * len(map_data_repr));\r\n            \r\n            for index_neighbour in map_data_repr[index + 1]:\r\n                data[index][index_neighbour - 1] = 1;\r\n    \r\n    file.close();\r\n    \r\n    # Set graph description\r\n    graph_descr = None;\r\n    if (data_type == 'm'): graph_descr = type_graph_descr.GRAPH_MATRIX_DESCR;\r\n    elif (data_type == 'v'): graph_descr = type_graph_descr.GRAPH_VECTOR_DESCR;\r\n    elif (data_type == 'e'): graph_descr = type_graph_descr.GRAPH_MATRIX_DESCR;\r\n    else:\r\n        raise NameError('Invalid format of file with graph description');\r\n    \r\n    if (space_descr != []):\r\n        if (len(data) != len(space_descr)):\r\n            raise NameError(\"Invalid format of file with graph - number of nodes is different in space representation and graph description\");\r\n    \r\n    return graph(data, graph_descr, space_descr, comments);"}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef draw_graph(graph_instance, map_coloring = None):\r\n    \r\n    if (graph_instance.space_description is None):\r\n        raise NameError(\"The graph haven't got representation in space\");\r\n    \r\n    if (map_coloring is not None):\r\n        if (len(graph_instance) != len(map_coloring)):\r\n            raise NameError(\"Size of graph should be equal to size coloring map\");\r\n        \r\n    \r\n    fig = plt.figure();\r\n    axes = fig.add_subplot(111);\r\n    \r\n    available_colors = ['#00a2e8', '#22b14c', '#ed1c24',\r\n                        '#fff200', '#000000', '#a349a4',\r\n                        '#ffaec9', '#7f7f7f', '#b97a57',\r\n                        '#c8bfe7', '#880015', '#ff7f27',\r\n                        '#3f48cc', '#c3c3c3', '#ffc90e',\r\n                        '#efe4b0', '#b5e61d', '#99d9ea',\r\n                        '#7092b4', '#ffffff'];\r\n              \r\n    if (map_coloring is not None):\r\n        if (len(map_coloring) > len(available_colors)):\r\n            raise NameError('Impossible to represent colored graph due to number of specified colors.');\r\n    \r\n    x_maximum = -float('inf');\r\n    x_minimum = float('inf');\r\n    y_maximum = -float('inf');\r\n    y_minimum = float('inf');\r\n    \r\n    for i in range(0, len(graph_instance.space_description), 1):\r\n        if (graph_instance.type_graph_descr == type_graph_descr.GRAPH_MATRIX_DESCR):\r\n            for j in range(i, len(graph_instance.space_description), 1):    # draw connection between two points only one time\r\n                if (graph_instance.data[i][j] == 1):\r\n                    axes.plot([graph_instance.space_description[i][0], graph_instance.space_description[j][0]], [graph_instance.space_description[i][1], graph_instance.space_description[j][1]], 'k-', linewidth = 1.5);\r\n                    \r\n        elif (graph_instance.type_graph_descr == type_graph_descr.GRAPH_VECTOR_DESCR):\r\n            for j in graph_instance.data[i]:\r\n                if (i > j):     # draw connection between two points only one time\r\n                    axes.plot([graph_instance.space_description[i][0], graph_instance.space_description[j][0]], [graph_instance.space_description[i][1], graph_instance.space_description[j][1]], 'k-', linewidth = 1.5);   \r\n            \r\n        color_node = 'b';\r\n        if (map_coloring is not None):\r\n            color_node = colors.hex2color(available_colors[map_coloring[i]]);\r\n        \r\n        axes.plot(graph_instance.space_description[i][0], graph_instance.space_description[i][1], color = color_node, marker = 'o', markersize = 20);\r\n    \r\n        if (x_maximum < graph_instance.space_description[i][0]): x_maximum = graph_instance.space_description[i][0];\r\n        if (x_minimum > graph_instance.space_description[i][0]): x_minimum = graph_instance.space_description[i][0];  \r\n        if (y_maximum < graph_instance.space_description[i][1]): y_maximum = graph_instance.space_description[i][1]; \r\n        if (y_minimum > graph_instance.space_description[i][1]): y_minimum = graph_instance.space_description[i][1];\r\n    \r\n    plt.xlim(x_minimum - 0.5, x_maximum + 0.5);\r\n    plt.ylim(y_minimum - 0.5, y_maximum + 0.5);\r\n    \r\n    plt.show();", "response": "Draws a single graph."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef process(self):\r\n        if self.__ccore is True:\r\n            self.__process_by_ccore()\r\n        else:\r\n            self.__process_by_python()\r\n\r\n        return self", "response": "A method that processes the cluster by the Fuzzy C - Means algorithm."}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\nprocess the cluster by C ++ implementation.", "response": "def __process_by_ccore(self):\r\n        \"\"\"!\r\n        @brief Performs cluster analysis using C/C++ implementation.\r\n\r\n        \"\"\"\r\n        result = wrapper.fcm_algorithm(self.__data, self.__centers, self.__m, self.__tolerance, self.__itermax)\r\n\r\n        self.__clusters = result[wrapper.fcm_package_indexer.INDEX_CLUSTERS]\r\n        self.__centers = result[wrapper.fcm_package_indexer.INDEX_CENTERS]\r\n        self.__membership = result[wrapper.fcm_package_indexer.INDEX_MEMBERSHIP]"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef __process_by_python(self):\r\n        self.__data = numpy.array(self.__data)\r\n        self.__centers = numpy.array(self.__centers)\r\n\r\n        self.__membership = numpy.zeros((len(self.__data), len(self.__centers)))\r\n\r\n        change = float('inf')\r\n        iteration = 0\r\n\r\n        while change > self.__tolerance and iteration < self.__itermax:\r\n            self.__update_membership()\r\n            centers = self.__calculate_centers()\r\n            change = self.__calculate_changes(centers)\r\n\r\n            self.__centers = centers\r\n            iteration += 1\r\n\r\n        self.__extract_clusters()", "response": "This method is used by the cluster analysis process_by_python method."}
{"SOURCE": "codesearchnet", "instruction": "Make a summary of the following Python 3 code\ndef __calculate_centers(self):\r\n        dimension = self.__data.shape[1]\r\n        centers = numpy.zeros((len(self.__centers), dimension))\r\n\r\n        for i in range(len(self.__centers)):\r\n            # multiplication '@' requires python version 3.5\r\n            centers[i] = numpy.divide(self.__membership[:, i] @ self.__data, numpy.sum(self.__membership[:, i]))\r\n\r\n        return centers", "response": "Calculate center using membership of each cluster."}
{"SOURCE": "codesearchnet", "instruction": "How would you explain what the following Python 3 function does\ndef __update_membership(self):\r\n        data_difference = numpy.zeros((len(self.__centers), len(self.__data)))\r\n\r\n        for i in range(len(self.__centers)):\r\n            data_difference[i] = numpy.sum(numpy.square(self.__data - self.__centers[i]), axis=1)\r\n\r\n        for i in range(len(self.__data)):\r\n            for j in range(len(self.__centers)):\r\n                divider = sum([pow(data_difference[j][i] / data_difference[k][i], self.__degree) for k in range(len(self.__centers)) if data_difference[k][i] != 0.0])\r\n\r\n                if divider != 0.0:\r\n                    self.__membership[i][j] = 1.0 / divider\r\n                else:\r\n                    self.__membership[i][j] = 1.0", "response": "Update membership for each point in line with current cluster centers."}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef __calculate_changes(self, updated_centers):\r\n        changes = numpy.sum(numpy.square(self.__centers - updated_centers), axis=1).T\r\n        return numpy.max(changes)", "response": "Calculate the change between the centers and the updated centers."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\ncollecting the best chromosome and its fitness function s value.", "response": "def collect_global_best(self, best_chromosome, best_fitness_function):\n        \"\"\"!\n        @brief Stores the best chromosome and its fitness function's value.\n        \n        @param[in] best_chromosome (list): The best chromosome that were observed.\n        @param[in] best_fitness_function (float): Fitness function value of the best chromosome.\n        \n        \"\"\"\n\n        if not self._need_global_best:\n            return\n\n        self._global_best_result['chromosome'].append(best_chromosome)\n        self._global_best_result['fitness_function'].append(best_fitness_function)"}
{"SOURCE": "codesearchnet", "instruction": "Implement a function in Python 3 to\ncollect the best chromosome for current specific iteration and its fitness function.", "response": "def collect_population_best(self, best_chromosome, best_fitness_function):\n        \"\"\"!\n        @brief Stores the best chromosome for current specific iteration and its fitness function's value.\n        \n        @param[in] best_chromosome (list): The best chromosome on specific iteration.\n        @param[in] best_fitness_function (float): Fitness function value of the chromosome.\n        \n        \"\"\"\n\n        if not self._need_population_best:\n            return\n\n        self._best_population_result['chromosome'].append(best_chromosome)\n        self._best_population_result['fitness_function'].append(best_fitness_function)"}
{"SOURCE": "codesearchnet", "instruction": "Explain what the following Python 3 code does\ndef collect_mean(self, fitness_functions):\n\n        if not self._need_mean_ff:\n            return\n\n        self._mean_ff_result.append(np.mean(fitness_functions))", "response": "Collect the average value of fitness functions among chromosomes on specific iteration."}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ndisplay the evolution of fitness function for the current best chromosome and the current best chromosome and the average value among all chromosomes.", "response": "def show_evolution(observer, start_iteration = 0, stop_iteration=None, ax=None, display=True):\n        \"\"\"!\n        @brief Displays evolution of fitness function for the best chromosome, for the current best chromosome and\n                average value among all chromosomes.\n        \n        @param[in] observer (ga_observer): Genetic algorithm observer that was used for collecting evolution in the algorithm and\n                    where whole required information for visualization is stored.\n        @param[in] start_iteration (uint): Iteration from that evolution should be shown.\n        @param[in] stop_iteration (uint): Iteration after that evolution shouldn't be shown.\n        @param[in] ax (Axes): Canvas where evolution should be displayed.\n        @param[in] display (bool): If 'True' then visualization of the evolution will be shown by the function.\n                    This argument should be 'False' if you want to add something else to the canvas and display it later.\n        \n        @return (Axis) Canvas where evolution was shown.\n        \n        \"\"\"\n        \n        if (ax is None):\n            _, ax = plt.subplots(1)\n            ax.set_title(\"Evolution\")\n        \n        if stop_iteration is None:\n            stop_iteration = len(observer)\n        \n        line_best, = ax.plot(observer.get_global_best()['fitness_function'][start_iteration:stop_iteration], 'r')\n        line_current, = ax.plot(observer.get_population_best()['fitness_function'][start_iteration:stop_iteration], 'k')\n        line_mean, = ax.plot(observer.get_mean_fitness_function()[start_iteration:stop_iteration], 'c')\n\n        if start_iteration < (stop_iteration - 1):\n            ax.set_xlim([start_iteration, (stop_iteration - 1)])\n        \n        ax.set_xlabel(\"Iteration\")\n        ax.set_ylabel(\"Fitness function\")\n        ax.legend([line_best, line_current, line_mean], [\"The best pop.\", \"Cur. best pop.\", \"Average\"], prop={'size': 10})\n        ax.grid()\n\n        if display is True:\n            plt.show()\n        \n        return ax"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script to\nshow allocated clusters by the genetic algorithm.", "response": "def show_clusters(data, observer, marker='.', markersize=None):\n        \"\"\"!\n        @brief Shows allocated clusters by the genetic algorithm.\n        \n        @param[in] data (list): Input data that was used for clustering process by the algorithm.\n        @param[in] observer (ga_observer): Observer that was used for collection information about clustering process.\n        @param[in] marker (char): Type of marker that should be used for object (point) representation.\n        @param[in] markersize (uint): Size of the marker that is used for object (point) representation.\n        \n        @note If you have clusters instead of observer then 'cluster_visualizer' can be used for visualization purposes.\n        \n        @see cluster_visualizer\n        \n        \"\"\"\n        \n        figure = plt.figure()\n        ax1 = figure.add_subplot(121)\n        \n        clusters = ga_math.get_clusters_representation(observer.get_global_best()['chromosome'][-1])\n        \n        visualizer = cluster_visualizer(1, 2)\n        visualizer.append_clusters(clusters, data, 0, marker, markersize)\n        visualizer.show(figure, display=False)\n        \n        ga_visualizer.show_evolution(observer, 0, None, ax1, True)"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 script for\nanimating clustering process of genetic clustering algorithm.", "response": "def animate_cluster_allocation(data, observer, animation_velocity=75, movie_fps=5, save_movie=None):\n        \"\"\"!\n        @brief Animate clustering process of genetic clustering algorithm.\n        @details This method can be also used for rendering movie of clustering process and 'ffmpeg' is required for that purpuse.\n        \n        @param[in] data (list): Input data that was used for clustering process by the algorithm.\n        @param[in] observer (ga_observer): Observer that was used for collection information about clustering process.\n                    Be sure that whole information was collected by the observer.\n        @param[in] animation_velocity (uint): Interval between frames in milliseconds (for run-time animation only).\n        @param[in] movie_fps (uint): Defines frames per second (for rendering movie only).\n        @param[in] save_movie (string): If it is specified then animation will be stored to file that is specified in this parameter.\n        \n        \"\"\"\n        \n        figure = plt.figure()\n        \n        def init_frame():\n            return frame_generation(0)\n\n        def frame_generation(index_iteration):\n            figure.clf()\n            \n            figure.suptitle(\"Clustering genetic algorithm (iteration: \" + str(index_iteration) + \")\", fontsize=18, fontweight='bold')\n            \n            visualizer = cluster_visualizer(4, 2, [\"The best pop. on step #\" + str(index_iteration), \"The best population\"])\n            \n            local_minimum_clusters = ga_math.get_clusters_representation(observer.get_population_best()['chromosome'][index_iteration])\n            visualizer.append_clusters(local_minimum_clusters, data, 0)\n            \n            global_minimum_clusters = ga_math.get_clusters_representation(observer.get_global_best()['chromosome'][index_iteration])\n            visualizer.append_clusters(global_minimum_clusters, data, 1)\n            \n            ax1 = plt.subplot2grid((2, 2), (1, 0), colspan=2)\n            ga_visualizer.show_evolution(observer, 0, index_iteration + 1, ax1, False)\n            \n            visualizer.show(figure, shift=0, display=False)\n            figure.subplots_adjust(top=0.85)\n            \n            return [figure.gca()]\n        \n        iterations = len(observer)\n        cluster_animation = animation.FuncAnimation(figure, frame_generation, iterations, interval=animation_velocity, init_func=init_frame, repeat_delay=5000)\n\n        if save_movie is not None:\n            cluster_animation.save(save_movie, writer='ffmpeg', fps=movie_fps, bitrate=1500)\n        else:\n            plt.show()"}
{"SOURCE": "codesearchnet", "instruction": "Given the following Python 3 function, write the documentation\ndef process(self):\n\n        # Initialize population\n        chromosomes = self._init_population(self._count_clusters, len(self._data), self._chromosome_count)\n\n        # Initialize the Best solution\n        best_chromosome, best_ff, first_fitness_functions \\\n            = self._get_best_chromosome(chromosomes, self._data, self._count_clusters)\n\n        # Save best result into observer\n        if self._observer is not None:\n            self._observer.collect_global_best(best_chromosome, best_ff)\n            self._observer.collect_population_best(best_chromosome, best_ff)\n            self._observer.collect_mean(first_fitness_functions)\n\n        # Next population\n        for _ in range(self._population_count):\n\n            # Select\n            chromosomes = self._select(chromosomes, self._data, self._count_clusters, self._select_coeff)\n\n            # Crossover\n            self._crossover(chromosomes)\n\n            # Mutation\n            self._mutation(chromosomes, self._count_clusters, self._count_mutation_gens, self._coeff_mutation_count)\n\n            # Update the Best Solution\n            new_best_chromosome, new_best_ff, fitness_functions \\\n                = self._get_best_chromosome(chromosomes, self._data, self._count_clusters)\n\n            # Get best chromosome\n            if new_best_ff < best_ff:\n                best_ff = new_best_ff\n                best_chromosome = new_best_chromosome\n\n            # Save best result into observer\n            if self._observer is not None:\n                self._observer.collect_global_best(best_chromosome, best_ff)\n                self._observer.collect_population_best(new_best_chromosome, new_best_ff)\n                self._observer.collect_mean(fitness_functions)\n\n        # Save result\n        self._result_clustering['best_chromosome'] = best_chromosome\n        self._result_clustering['best_fitness_function'] = best_ff\n\n        return best_chromosome, best_ff", "response": "This method processes the genetic clustering algorithm in line with rule of genetic clustering algorithm."}
{"SOURCE": "codesearchnet", "instruction": "Implement a Python 3 function for\nselecting new chromosomes based on the given data.", "response": "def _select(chromosomes, data, count_clusters, select_coeff):\n        \"\"\"!\n        @brief Performs selection procedure where new chromosomes are calculated.\n        \n        @param[in] chromosomes (numpy.array): Chromosomes \n        \n        \"\"\"\n\n        # Calc centers\n        centres = ga_math.get_centres(chromosomes, data, count_clusters)\n\n        # Calc fitness functions\n        fitness = genetic_algorithm._calc_fitness_function(centres, data, chromosomes)\n\n        for _idx in range(len(fitness)):\n            fitness[_idx] = math.exp(1 + fitness[_idx] * select_coeff)\n\n        # Calc probability vector\n        probabilities = ga_math.calc_probability_vector(fitness)\n\n        # Select P chromosomes with probabilities\n        new_chromosomes = np.zeros(chromosomes.shape, dtype=np.int)\n\n        # Selecting\n        for _idx in range(len(chromosomes)):\n            new_chromosomes[_idx] = chromosomes[ga_math.get_uniform(probabilities)]\n\n        return new_chromosomes"}
{"SOURCE": "codesearchnet", "instruction": "Can you tell what is the following Python 3 function doing\ndef _mutation(chromosomes, count_clusters, count_gen_for_mutation, coeff_mutation_count):\n\n        # Count gens in Chromosome\n        count_gens = len(chromosomes[0])\n\n        # Get random chromosomes for mutation\n        random_idx_chromosomes = np.array(range(len(chromosomes)))\n        np.random.shuffle(random_idx_chromosomes)\n\n        #\n        for _idx_chromosome in range(int(len(random_idx_chromosomes) * coeff_mutation_count)):\n\n            #\n            for _ in range(count_gen_for_mutation):\n\n                # Get random gen\n                gen_num = np.random.randint(count_gens)\n\n                # Set random cluster\n                chromosomes[random_idx_chromosomes[_idx_chromosome]][gen_num] = np.random.randint(count_clusters)", "response": "This function is used to generate random cluster and mutation chromosomes."}
{"SOURCE": "codesearchnet", "instruction": "Create a Python 3 function to\nreturn a crossover mask that can be used to crossover a pair of chromosomes.", "response": "def _get_crossover_mask(mask_length):\n        \"\"\"!\n        @brief Crossover mask to crossover a pair of chromosomes.\n        \n        @param[in] mask_length (uint): Length of the mask.\n        \n        \"\"\"\n\n        # Initialize mask\n        mask = np.zeros(mask_length)\n\n        # Set a half of array to 1\n        mask[:int(int(mask_length) / 2)] = 1\n\n        # Random shuffle\n        np.random.shuffle(mask)\n\n        return mask"}
{"SOURCE": "codesearchnet", "instruction": "Write a Python 3 function that can\ninitialize the first population as a uniform random choice.", "response": "def _init_population(count_clusters, count_data, chromosome_count):\n        \"\"\"!\n        @brief Returns first population as a uniform random choice.\n        \n        @param[in] count_clusters (uint): Amount of clusters that should be allocated.\n        @param[in] count_data (uint): Data size that is used for clustering process.\n        @param[in] chromosome_count (uint):Amount of chromosome that is used for clustering.\n        \n        \"\"\"\n\n        population = np.random.randint(count_clusters, size=(chromosome_count, count_data))\n\n        return population"}
{"SOURCE": "codesearchnet", "instruction": "Can you generate the documentation for the following Python 3 function\ndef _calc_fitness_function(centres, data, chromosomes):\n\n        # Get count of chromosomes and clusters\n        count_chromosome = len(chromosomes)\n\n        # Initialize fitness function values\n        fitness_function = np.zeros(count_chromosome)\n\n        # Calc fitness function for each chromosome\n        for _idx_chromosome in range(count_chromosome):\n\n            # Get centers for a selected chromosome\n            centres_data = np.zeros(data.shape)\n\n            # Fill data centres\n            for _idx in range(len(data)):\n                centres_data[_idx] = centres[_idx_chromosome][chromosomes[_idx_chromosome][_idx]]\n\n            # Get City Block distance for a chromosome\n            fitness_function[_idx_chromosome] += np.sum(abs(data - centres_data))\n\n        return fitness_function", "response": "Calculates the fitness function values for each chromosome correspondingly."}
{"SOURCE": "codesearchnet", "instruction": "Can you implement a function in Python 3 that\ngets the distance between two clusters in line with the given type.", "response": "def get_distance(self, entry, type_measurement):\r\n        \"\"\"!\r\n        @brief Calculates distance between two clusters in line with measurement type.\r\n        \r\n        @details In case of usage CENTROID_EUCLIDIAN_DISTANCE square euclidian distance will be returned.\r\n                 Square root should be taken from the result for obtaining real euclidian distance between\r\n                 entries. \r\n        \r\n        @param[in] entry (cfentry): Clustering feature to which distance should be obtained.\r\n        @param[in] type_measurement (measurement_type): Distance measurement algorithm between two clusters.\r\n        \r\n        @return (double) Distance between two clusters.\r\n        \r\n        \"\"\"\r\n        \r\n        if (type_measurement is measurement_type.CENTROID_EUCLIDEAN_DISTANCE):\r\n            return euclidean_distance_square(entry.get_centroid(), self.get_centroid());\r\n        \r\n        elif (type_measurement is measurement_type.CENTROID_MANHATTAN_DISTANCE):\r\n            return manhattan_distance(entry.get_centroid(), self.get_centroid());\r\n        \r\n        elif (type_measurement is measurement_type.AVERAGE_INTER_CLUSTER_DISTANCE):\r\n            return self.__get_average_inter_cluster_distance(entry);\r\n            \r\n        elif (type_measurement is measurement_type.AVERAGE_INTRA_CLUSTER_DISTANCE):\r\n            return self.__get_average_intra_cluster_distance(entry);\r\n        \r\n        elif (type_measurement is measurement_type.VARIANCE_INCREASE_DISTANCE):\r\n            return self.__get_variance_increase_distance(entry);\r\n        \r\n        else:\r\n            assert 0;"}